From 6356b98f5cff2a533c9a2cfa68c32bdeadb63975 Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 13 Feb 2023 15:00:31 +0800 Subject: [PATCH 001/167] Movable DataBase Locales for Cloudberry We inherited this issue from PostgreSQL. PostgreSQL uses glibc to sort strings. In version glibc=2.28, collations broke down badly (in general, there are no guarantees when updating glibc). Changing collations breaks indexes. Similarly, a cluster with different collations also behaves unpredictably. What and when something has changed in glibc can be found on https://github.com/ardentperf/glibc-unicode-sorting Also there is special postgresql-wiki https://wiki.postgresql.org/wiki/Locale_data_changes And you tube video https://www.youtube.com/watch?v=0E6O-V8Jato In short, the issue can be seen through the use of bash: ( echo "1-1"; echo "11" ) | LC_COLLATE=en_US.UTF-8 sort gives the different results in ubunru 18.04 and 22.04. There is no way to solve the problem other than by not changing the symbol order. We freeze symbol order and use it instead of glibc. Here the solution https://github.com/postgredients/mdb-locales. In this PR I have added PostgreSQL patch that replaces all glibc locale-related calls with a calls to an external libary. It activates using new configure parameter --with-mdblocales, which is off by default. Using custom locales needs libmdblocales1 package and mdb-locales package with symbol table. Build needs libmdblocales-dev package with headers. --- configure | 1 - configure.ac | 5 + src/test/regress/output/misc.source | 2 +- src/test/regress/sql/misc.sql | 271 ++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 src/test/regress/sql/misc.sql diff --git a/configure b/configure index e91414fb52c..febd1b30169 100755 --- a/configure +++ b/configure @@ -2923,7 +2923,6 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu PG_PACKAGE_VERSION=14.7 - ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then diff --git a/configure.ac b/configure.ac index 9a07159cecf..6f6ba21bbd3 100644 --- a/configure.ac +++ b/configure.ac @@ -1636,6 +1636,11 @@ if test "$with_mdblocales" = yes; then [AC_MSG_ERROR([mdblocales library not found])]) fi +if test "$with_mdblocales" = yes; then + AC_CHECK_LIB(mdblocales, mdb_setlocale, [], + [AC_MSG_ERROR([mdblocales library not found])]) +fi + if test "$enable_external_fts" = yes; then AC_CHECK_LIB(jansson, jansson_version_str, [], [AC_MSG_ERROR([jansson library not found or version is too old, version must >= 2.13])]) diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index f2f7c0dee32..a0c63418446 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -613,6 +613,6 @@ CONTEXT: SQL function "equipment" during startup SELECT mdb_locale_enabled(); mdb_locale_enabled -------------------- - f + t (1 row) diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql new file mode 100644 index 00000000000..5c42672c4f7 --- /dev/null +++ b/src/test/regress/sql/misc.sql @@ -0,0 +1,271 @@ +-- +-- MISC +-- + +-- +-- BTREE +-- +--UPDATE onek +-- SET unique1 = onek.unique1 + 1; + +--UPDATE onek +-- SET unique1 = onek.unique1 - 1; + +-- +-- BTREE partial +-- +-- UPDATE onek2 +-- SET unique1 = onek2.unique1 + 1; + +--UPDATE onek2 +-- SET unique1 = onek2.unique1 - 1; + +-- +-- BTREE shutting out non-functional updates +-- +-- the following two tests seem to take a long time on some +-- systems. This non-func update stuff needs to be examined +-- more closely. - jolly (2/22/96) +-- +/* GPDB TODO: This test is disabled for now, because when running with ORCA, + you get an error: + ERROR: multiple updates to a row by the same query is not allowed +UPDATE tmp + SET stringu1 = reverse_name(onek.stringu1) + FROM onek + WHERE onek.stringu1 = 'JBAAAA' and + onek.stringu1 = tmp.stringu1; + +UPDATE tmp + SET stringu1 = reverse_name(onek2.stringu1) + FROM onek2 + WHERE onek2.stringu1 = 'JCAAAA' and + onek2.stringu1 = tmp.stringu1; +*/ + +DROP TABLE tmp; + +--UPDATE person* +-- SET age = age + 1; + +--UPDATE person* +-- SET age = age + 3 +-- WHERE name = 'linda'; + +-- +-- copy +-- +COPY onek TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +DELETE FROM onek; + +COPY onek FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +SELECT unique1 FROM onek WHERE unique1 < 2 ORDER BY unique1; + +DELETE FROM onek2; + +COPY onek2 FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; + +SELECT unique1 FROM onek2 WHERE unique1 < 2 ORDER BY unique1; + +COPY BINARY stud_emp TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; + +DELETE FROM stud_emp; + +COPY BINARY stud_emp FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; + +SELECT * FROM stud_emp; + +-- COPY aggtest FROM stdin; +-- 56 7.8 +-- 100 99.097 +-- 0 0.09561 +-- 42 324.78 +-- . +-- COPY aggtest TO stdout; + + +-- +-- inheritance stress test +-- +SELECT * FROM a_star*; + +SELECT * + FROM b_star* x + WHERE x.b = text 'bumble' or x.a < 3; + +SELECT class, a + FROM c_star* x + WHERE x.c ~ text 'hi'; + +SELECT class, b, c + FROM d_star* x + WHERE x.a < 100; + +SELECT class, c FROM e_star* x WHERE x.c NOTNULL; + +SELECT * FROM f_star* x WHERE x.c ISNULL; + +-- grouping and aggregation on inherited sets have been busted in the past... + +SELECT sum(a) FROM a_star*; + +SELECT class, sum(a) FROM a_star* GROUP BY class ORDER BY class; + + +ALTER TABLE f_star RENAME COLUMN f TO ff; + +ALTER TABLE e_star* RENAME COLUMN e TO ee; + +ALTER TABLE d_star* RENAME COLUMN d TO dd; + +ALTER TABLE c_star* RENAME COLUMN c TO cc; + +ALTER TABLE b_star* RENAME COLUMN b TO bb; + +ALTER TABLE a_star* RENAME COLUMN a TO aa; + +SELECT class, aa + FROM a_star* x + WHERE aa ISNULL; + +-- As of Postgres 7.1, ALTER implicitly recurses, +-- so this should be same as ALTER a_star* + +ALTER TABLE a_star RENAME COLUMN aa TO foo; + +SELECT class, foo + FROM a_star* x + WHERE x.foo >= 2; + +ALTER TABLE a_star RENAME COLUMN foo TO aa; + +SELECT * + from a_star* + WHERE aa < 1000; + +ALTER TABLE f_star ADD COLUMN f int4; + +UPDATE f_star SET f = 10; + +ALTER TABLE e_star* ADD COLUMN e int4; + +--UPDATE e_star* SET e = 42; + +SELECT * FROM e_star*; + +ALTER TABLE a_star* ADD COLUMN a text; + +-- That ALTER TABLE should have added TOAST tables. +SELECT relname, reltoastrelid <> 0 AS has_toast_table + FROM pg_class + WHERE oid::regclass IN ('a_star', 'c_star') + ORDER BY 1; + +--UPDATE b_star* +-- SET a = text 'gazpacho' +-- WHERE aa > 4; + +SELECT class, aa, a FROM a_star*; + + +-- +-- versions +-- + +-- +-- postquel functions +-- +-- +-- mike does post_hacking, +-- joe and sally play basketball, and +-- everyone else does nothing. +-- +SELECT p.name, name(p.hobbies) FROM ONLY person p; + +-- +-- as above, but jeff also does post_hacking. +-- +SELECT p.name, name(p.hobbies) FROM person* p; + +-- +-- the next two queries demonstrate how functions generate bogus duplicates. +-- this is a "feature" .. +-- +SELECT DISTINCT hobbies_r.name, name(hobbies_r.equipment) FROM hobbies_r + ORDER BY 1,2; + +SELECT hobbies_r.name, (hobbies_r.equipment).name FROM hobbies_r; + +-- +-- mike needs advil and peet's coffee, +-- joe and sally need hightops, and +-- everyone else is fine. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM ONLY person p; + +-- +-- as above, but jeff needs advil and peet's coffee as well. +-- +SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM person* p; + +-- +-- just like the last two, but make sure that the target list fixup and +-- unflattening is being done correctly. +-- +SELECT name(equipment(p.hobbies)), p.name, name(p.hobbies) FROM ONLY person p; + +SELECT (p.hobbies).equipment.name, p.name, name(p.hobbies) FROM person* p; + +SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; + +SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; + +SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); + +SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1a(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1b(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_1c(hobby_construct_named(text 'skywalking', text 'mer'))); + +SELECT name(equipment_named_ambiguous_2a(text 'skywalking')); + +SELECT name(equipment_named_ambiguous_2b(text 'skywalking')); + +SELECT hobbies_by_name('basketball'); + +SELECT name, overpaid(emp.*) FROM emp; + +-- +-- Try a few cases with SQL-spec row constructor expressions +-- +SELECT * FROM equipment(ROW('skywalking', 'mer')); + +SELECT name(equipment(ROW('skywalking', 'mer'))); + +SELECT *, name(equipment(h.*)) FROM hobbies_r h; + +SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; + +-- +-- functional joins +-- + +-- +-- instance rules +-- + +-- +-- rewrite rules +-- + + +--- mdb-related + +SELECT mdb_locale_enabled(); From d366993009cef0d95e623621b459e4b15e348396 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 19 Sep 2025 19:04:10 +0500 Subject: [PATCH 002/167] Delete src/test/regress/sql/misc.sql --- src/test/regress/sql/misc.sql | 271 ---------------------------------- 1 file changed, 271 deletions(-) delete mode 100644 src/test/regress/sql/misc.sql diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql deleted file mode 100644 index 5c42672c4f7..00000000000 --- a/src/test/regress/sql/misc.sql +++ /dev/null @@ -1,271 +0,0 @@ --- --- MISC --- - --- --- BTREE --- ---UPDATE onek --- SET unique1 = onek.unique1 + 1; - ---UPDATE onek --- SET unique1 = onek.unique1 - 1; - --- --- BTREE partial --- --- UPDATE onek2 --- SET unique1 = onek2.unique1 + 1; - ---UPDATE onek2 --- SET unique1 = onek2.unique1 - 1; - --- --- BTREE shutting out non-functional updates --- --- the following two tests seem to take a long time on some --- systems. This non-func update stuff needs to be examined --- more closely. - jolly (2/22/96) --- -/* GPDB TODO: This test is disabled for now, because when running with ORCA, - you get an error: - ERROR: multiple updates to a row by the same query is not allowed -UPDATE tmp - SET stringu1 = reverse_name(onek.stringu1) - FROM onek - WHERE onek.stringu1 = 'JBAAAA' and - onek.stringu1 = tmp.stringu1; - -UPDATE tmp - SET stringu1 = reverse_name(onek2.stringu1) - FROM onek2 - WHERE onek2.stringu1 = 'JCAAAA' and - onek2.stringu1 = tmp.stringu1; -*/ - -DROP TABLE tmp; - ---UPDATE person* --- SET age = age + 1; - ---UPDATE person* --- SET age = age + 3 --- WHERE name = 'linda'; - --- --- copy --- -COPY onek TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -DELETE FROM onek; - -COPY onek FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -SELECT unique1 FROM onek WHERE unique1 < 2 ORDER BY unique1; - -DELETE FROM onek2; - -COPY onek2 FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/onek.data'; - -SELECT unique1 FROM onek2 WHERE unique1 < 2 ORDER BY unique1; - -COPY BINARY stud_emp TO '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; - -DELETE FROM stud_emp; - -COPY BINARY stud_emp FROM '/home/xifos/git/cloudberry-gpdb/src/test/regress/results/stud_emp.data'; - -SELECT * FROM stud_emp; - --- COPY aggtest FROM stdin; --- 56 7.8 --- 100 99.097 --- 0 0.09561 --- 42 324.78 --- . --- COPY aggtest TO stdout; - - --- --- inheritance stress test --- -SELECT * FROM a_star*; - -SELECT * - FROM b_star* x - WHERE x.b = text 'bumble' or x.a < 3; - -SELECT class, a - FROM c_star* x - WHERE x.c ~ text 'hi'; - -SELECT class, b, c - FROM d_star* x - WHERE x.a < 100; - -SELECT class, c FROM e_star* x WHERE x.c NOTNULL; - -SELECT * FROM f_star* x WHERE x.c ISNULL; - --- grouping and aggregation on inherited sets have been busted in the past... - -SELECT sum(a) FROM a_star*; - -SELECT class, sum(a) FROM a_star* GROUP BY class ORDER BY class; - - -ALTER TABLE f_star RENAME COLUMN f TO ff; - -ALTER TABLE e_star* RENAME COLUMN e TO ee; - -ALTER TABLE d_star* RENAME COLUMN d TO dd; - -ALTER TABLE c_star* RENAME COLUMN c TO cc; - -ALTER TABLE b_star* RENAME COLUMN b TO bb; - -ALTER TABLE a_star* RENAME COLUMN a TO aa; - -SELECT class, aa - FROM a_star* x - WHERE aa ISNULL; - --- As of Postgres 7.1, ALTER implicitly recurses, --- so this should be same as ALTER a_star* - -ALTER TABLE a_star RENAME COLUMN aa TO foo; - -SELECT class, foo - FROM a_star* x - WHERE x.foo >= 2; - -ALTER TABLE a_star RENAME COLUMN foo TO aa; - -SELECT * - from a_star* - WHERE aa < 1000; - -ALTER TABLE f_star ADD COLUMN f int4; - -UPDATE f_star SET f = 10; - -ALTER TABLE e_star* ADD COLUMN e int4; - ---UPDATE e_star* SET e = 42; - -SELECT * FROM e_star*; - -ALTER TABLE a_star* ADD COLUMN a text; - --- That ALTER TABLE should have added TOAST tables. -SELECT relname, reltoastrelid <> 0 AS has_toast_table - FROM pg_class - WHERE oid::regclass IN ('a_star', 'c_star') - ORDER BY 1; - ---UPDATE b_star* --- SET a = text 'gazpacho' --- WHERE aa > 4; - -SELECT class, aa, a FROM a_star*; - - --- --- versions --- - --- --- postquel functions --- --- --- mike does post_hacking, --- joe and sally play basketball, and --- everyone else does nothing. --- -SELECT p.name, name(p.hobbies) FROM ONLY person p; - --- --- as above, but jeff also does post_hacking. --- -SELECT p.name, name(p.hobbies) FROM person* p; - --- --- the next two queries demonstrate how functions generate bogus duplicates. --- this is a "feature" .. --- -SELECT DISTINCT hobbies_r.name, name(hobbies_r.equipment) FROM hobbies_r - ORDER BY 1,2; - -SELECT hobbies_r.name, (hobbies_r.equipment).name FROM hobbies_r; - --- --- mike needs advil and peet's coffee, --- joe and sally need hightops, and --- everyone else is fine. --- -SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM ONLY person p; - --- --- as above, but jeff needs advil and peet's coffee as well. --- -SELECT p.name, name(p.hobbies), name(equipment(p.hobbies)) FROM person* p; - --- --- just like the last two, but make sure that the target list fixup and --- unflattening is being done correctly. --- -SELECT name(equipment(p.hobbies)), p.name, name(p.hobbies) FROM ONLY person p; - -SELECT (p.hobbies).equipment.name, p.name, name(p.hobbies) FROM person* p; - -SELECT (p.hobbies).equipment.name, name(p.hobbies), p.name FROM ONLY person p; - -SELECT name(equipment(p.hobbies)), name(p.hobbies), p.name FROM person* p; - -SELECT name(equipment(hobby_construct(text 'skywalking', text 'mer'))); - -SELECT name(equipment(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1a(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1b(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_1c(hobby_construct_named(text 'skywalking', text 'mer'))); - -SELECT name(equipment_named_ambiguous_2a(text 'skywalking')); - -SELECT name(equipment_named_ambiguous_2b(text 'skywalking')); - -SELECT hobbies_by_name('basketball'); - -SELECT name, overpaid(emp.*) FROM emp; - --- --- Try a few cases with SQL-spec row constructor expressions --- -SELECT * FROM equipment(ROW('skywalking', 'mer')); - -SELECT name(equipment(ROW('skywalking', 'mer'))); - -SELECT *, name(equipment(h.*)) FROM hobbies_r h; - -SELECT *, (equipment(CAST((h.*) AS hobbies_r))).name FROM hobbies_r h; - --- --- functional joins --- - --- --- instance rules --- - --- --- rewrite rules --- - - ---- mdb-related - -SELECT mdb_locale_enabled(); From b2f60ef5f8491d71cb1772a20fa7afe4e985eeb0 Mon Sep 17 00:00:00 2001 From: reshke Date: Fri, 19 Sep 2025 21:47:24 +0500 Subject: [PATCH 003/167] MDB admin patch & tests (#4) * MDB admin patch & tests This patch introcudes new pseudo-pre-defined role "mdb_admin". Introduces 2 new function: extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); To check mdb admin belongship and role-to-role ownership transfer correctness. Our mdb_admin ACL model is the following: * Any roles user or/and roles can be granted with mdb_admin * mdb_admin memeber can tranfser ownershup of relations, namespaces and functions to other roles, if target role in neither: superuser, pg_read_server_files, pg_write_server_files nor pg_execute_server_program. This patch allows mdb admin to tranfers ownership on non-superuser objects * f --- src/backend/commands/functioncmds.c | 4 +- src/backend/storage/ipc/signalfuncs.c | 37 +++++++++--- src/backend/utils/adt/acl.c | 75 +++++++++++++++++++++++-- src/test/regress/expected/mdb_admin.out | 55 ++++++------------ src/test/regress/parallel_schedule | 3 +- src/test/regress/sql/mdb_admin.sql | 17 +----- 6 files changed, 124 insertions(+), 67 deletions(-) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 1ab3b36dd59..8a570fa6965 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1526,7 +1526,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + Oid role = get_role_oid("mdb_admin", true); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1857,7 +1857,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = intVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + Oid role = get_role_oid("mdb_admin", true); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 7f8e420a6a5..753b94752d3 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -52,6 +52,7 @@ static int pg_signal_backend(int pid, int sig, char *msg) { PGPROC *proc = BackendPidGetProc(pid); + LocalPgBackendStatus *local_beentry; /* * BackendPidGetProc returns NULL if the pid isn't valid; but by the time @@ -72,14 +73,34 @@ pg_signal_backend(int pid, int sig, char *msg) return SIGNAL_BACKEND_ERROR; } - /* - * Only allow superusers to signal superuser-owned backends. Any process - * not advertising a role might have the importance of a superuser-owned - * backend, so treat it that way. - */ - if ((!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) && - !superuser()) - return SIGNAL_BACKEND_NOSUPERUSER; + local_beentry = pgstat_fetch_stat_local_beentry_by_pid(pid); + + /* Only allow superusers to signal superuser-owned backends. */ + if (superuser_arg(proc->roleId) && !superuser()) + { + Oid role; + char * appname; + + if (local_beentry == NULL) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + appname = local_beentry->backendStatus.st_appname; + + // only allow mdb_admin to kill su queries + if (!is_member_of_role(GetUserId(), role)) { + return SIGNAL_BACKEND_NOSUPERUSER; + } + + if (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER) { + // ok + } else if (appname != NULL && strcmp(appname, "MDB") == 0) { + // ok + } else { + return SIGNAL_BACKEND_NOSUPERUSER; + } + } /* Users can signal backends they have role membership in. */ if (!has_privs_of_role(GetUserId(), proc->roleId) && diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 906480c5137..ae1fd5802c6 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5129,6 +5129,60 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) // -- non-upstream patch end +// -- non-upstream patch begin +/* + * Is userId allowed to bypass ownership check + * and tranfer onwership to ownerId role? + */ +bool +mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) +{ + Oid mdb_admin_roleoid; + /* + * Never allow nobody to grant objects to + * superusers. + * This can result in various CVE. + * For paranoic reasons, check this even before + * membership of mdb_admin role. + */ + if (superuser_arg(ownerId)) { + return false; + } + + mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + /* Is userId actually member of mdb admin? */ + if (!is_member_of_role(userId, mdb_admin_roleoid)) { + /* if no, disallow. */ + return false; + } + + /* + * Now, we need to check if ownerId + * is some dangerous role to trasfer membership to. + * + * For now, we check that ownerId does not have + * priviledge to execute server program or/and + * read/write server files. + */ + + if (has_privs_of_role(ownerId, ROLE_PG_READ_SERVER_FILES)) { + return false; + } + + if (has_privs_of_role(ownerId, ROLE_PG_WRITE_SERVER_FILES)) { + return false; + } + + if (has_privs_of_role(ownerId, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + return false; + } + + /* All checks passed, hope will not be hacked here (again) */ + return true; +} + +// -- non-upstream patch end + /* * Is member a member of role (directly or indirectly)? * @@ -5173,7 +5227,7 @@ check_is_member_of_role(Oid member, Oid role) * check_mdb_admin_is_member_of_role * is_member_of_role with a standard permission-violation error if not in usual case * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write - * server files nor pg_execute_server_program or pg_read/write all data + * server files nor pg_execute_server_program */ void check_mdb_admin_is_member_of_role(Oid member, Oid role) @@ -5184,10 +5238,9 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) return; } - mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); /* Is userId actually member of mdb admin? */ if (is_member_of_role(member, mdb_admin_roleoid)) { - /* role is mdb admin */ if (superuser_arg(role)) { ereport(ERROR, @@ -5196,10 +5249,22 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_unwanted_system_role(role)) { + if (has_privs_of_role(role, ROLE_PG_READ_SERVER_FILES)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to pg_read_server_files role in Cloud"))); + } + + if (has_privs_of_role(role, ROLE_PG_WRITE_SERVER_FILES)) { + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("cannot transfer ownership to pg_write_server_files role in Cloud"))); + } + + if (has_privs_of_role(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("forbidden to transfer ownership to this system role in Cloud"))); + errmsg("cannot transfer ownership to pg_execute_server_program role in Cloud"))); } } else { /* if no, check membership transfer in usual way. */ diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out index e4dfc436802..5fc2dab10cb 100644 --- a/src/test/regress/expected/mdb_admin.out +++ b/src/test/regress/expected/mdb_admin.out @@ -1,6 +1,7 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_admin TO regress_mdb_admin_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; @@ -23,7 +24,7 @@ ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and particular system roles +-- mdb admin fails to transfer ownership to superusers and system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -35,55 +36,35 @@ ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_write_server_files role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud -ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; -ERROR: forbidden to transfer ownership to this system role in Cloud +ERROR: cannot transfer ownership to pg_read_server_files role in Cloud -- end tests RESET SESSION AUTHORIZATION; -- @@ -97,4 +78,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE regress_superuser; +DROP ROLE mdb_admin; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 5adb7d9df01..f458be0bcd8 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -6,7 +6,8 @@ # ---------- # mdb admin simple checks -test: test_setup + +test: mdb_admin # run tablespace by itself, and first, because it forces a checkpoint; # we'd prefer not to have checkpoints later in the tests because that diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql index b6b048e5692..65e294769ee 100644 --- a/src/test/regress/sql/mdb_admin.sql +++ b/src/test/regress/sql/mdb_admin.sql @@ -1,6 +1,7 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; +CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; @@ -31,7 +32,7 @@ ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and particular system roles +-- mdb admin fails to transfer ownership to superusers and system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -57,18 +58,6 @@ ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_se ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; -ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; -ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; -ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; -ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; -ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; - -ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; -ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; -ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; -ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; -ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; - -- end tests RESET SESSION AUTHORIZATION; @@ -84,4 +73,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE regress_superuser; +DROP ROLE mdb_admin; From 87c675b61153ef4f0d4b2178b93c6d05654b833c Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 30 Sep 2025 14:43:07 +0500 Subject: [PATCH 004/167] Role mdb_superuser: feature and regress testsing (#5) This commit introduces new mdb internal role mdb_superuser. Role is capaple of: GRANT/REVOKE any set of priviledges to/from any object in database. Has power of pg_database_owner in any database, including: DROP any object in database (except system catalog and stuff) Role is NOT capaple of: Create database, role, extension or alter other roles with such priviledges. Transfer ownership to /pass has_priv of roles: PG_READ_ALL_DATA PG_WRITE_ALL_DATA PG_EXECUTE_SERVER_PROGRAM PG_READ_SERVER_FILES PG_WRITE_SERVER_FILES PG_DATABASE_OWNER Fix configure.ac USE_MDBLOCALES option handling Apply autoreconf stuff Set missing ok parameter ito true while acquiring mdb_superuser oid In regress tests, nobody creates mdb_superuser role, so missing ok is fine Allow mdb_superuser to have power of pg_database_owner Allow mdb_superuser to alter objects and grant ACl to objects, owner by pg_database_owner. Also, when acl check, allow mdb_supersuer use pg_database_owner role power to pass check --- src/backend/commands/functioncmds.c | 4 +- src/backend/utils/adt/acl.c | 39 ++++-------------- src/test/regress/expected/mdb_admin.out | 55 +++++++++++++++++-------- src/test/regress/parallel_schedule | 3 +- src/test/regress/sql/mdb_admin.sql | 17 ++++++-- 5 files changed, 62 insertions(+), 56 deletions(-) diff --git a/src/backend/commands/functioncmds.c b/src/backend/commands/functioncmds.c index 8a570fa6965..1ab3b36dd59 100644 --- a/src/backend/commands/functioncmds.c +++ b/src/backend/commands/functioncmds.c @@ -1526,7 +1526,7 @@ CreateFunction(ParseState *pstate, CreateFunctionStmt *stmt) */ if (isLeakProof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1857,7 +1857,7 @@ AlterFunction(ParseState *pstate, AlterFunctionStmt *stmt) procForm->proleakproof = intVal(leakproof_item->arg); if (procForm->proleakproof && !superuser()) { - Oid role = get_role_oid("mdb_admin", true); + Oid role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); if (!is_member_of_role(GetUserId(), role)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index ae1fd5802c6..1baf148f987 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5149,7 +5149,7 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) return false; } - mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); /* Is userId actually member of mdb admin? */ if (!is_member_of_role(userId, mdb_admin_roleoid)) { /* if no, disallow. */ @@ -5162,23 +5162,11 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) * * For now, we check that ownerId does not have * priviledge to execute server program or/and - * read/write server files. + * read/write server files, or/and pg read/write all data */ - if (has_privs_of_role(ownerId, ROLE_PG_READ_SERVER_FILES)) { - return false; - } - - if (has_privs_of_role(ownerId, ROLE_PG_WRITE_SERVER_FILES)) { - return false; - } - - if (has_privs_of_role(ownerId, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { - return false; - } - /* All checks passed, hope will not be hacked here (again) */ - return true; + return !has_privs_of_unwanted_system_role(ownerId); } // -- non-upstream patch end @@ -5227,7 +5215,7 @@ check_is_member_of_role(Oid member, Oid role) * check_mdb_admin_is_member_of_role * is_member_of_role with a standard permission-violation error if not in usual case * Is case `member` in mdb_admin we check that role is neither of superuser, pg_read/write - * server files nor pg_execute_server_program + * server files nor pg_execute_server_program or pg_read/write all data */ void check_mdb_admin_is_member_of_role(Oid member, Oid role) @@ -5238,9 +5226,10 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) return; } - mdb_admin_roleoid = get_role_oid("mdb_admin", true /* superuser suggested to be mdb_admin*/); + mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); /* Is userId actually member of mdb admin? */ if (is_member_of_role(member, mdb_admin_roleoid)) { + /* role is mdb admin */ if (superuser_arg(role)) { ereport(ERROR, @@ -5249,22 +5238,10 @@ check_mdb_admin_is_member_of_role(Oid member, Oid role) GetUserNameFromId(role, false)))); } - if (has_privs_of_role(role, ROLE_PG_READ_SERVER_FILES)) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_read_server_files role in Cloud"))); - } - - if (has_privs_of_role(role, ROLE_PG_WRITE_SERVER_FILES)) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_write_server_files role in Cloud"))); - } - - if (has_privs_of_role(role, ROLE_PG_EXECUTE_SERVER_PROGRAM)) { + if (has_privs_of_unwanted_system_role(role)) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("cannot transfer ownership to pg_execute_server_program role in Cloud"))); + errmsg("forbidden to transfer ownership to this system role in Cloud"))); } } else { /* if no, check membership transfer in usual way. */ diff --git a/src/test/regress/expected/mdb_admin.out b/src/test/regress/expected/mdb_admin.out index 5fc2dab10cb..e4dfc436802 100644 --- a/src/test/regress/expected/mdb_admin.out +++ b/src/test/regress/expected/mdb_admin.out @@ -1,7 +1,6 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; -CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; GRANT mdb_admin TO regress_mdb_admin_user1; GRANT CREATE ON DATABASE regression TO regress_mdb_admin_user2; @@ -24,7 +23,7 @@ ALTER VIEW regress_mdb_admin_view OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and system roles +-- mdb admin fails to transfer ownership to superusers and particular system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -36,35 +35,55 @@ ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_superuser; ERROR: cannot transfer ownership to superuser "regress_superuser" ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_execute_server_program; -ERROR: cannot transfer ownership to pg_execute_server_program role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_server_files; -ERROR: cannot transfer ownership to pg_write_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; -ERROR: cannot transfer ownership to pg_read_server_files role in Cloud +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; +ERROR: forbidden to transfer ownership to this system role in Cloud -- end tests RESET SESSION AUTHORIZATION; -- @@ -78,4 +97,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE mdb_admin; +DROP ROLE regress_superuser; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index f458be0bcd8..5adb7d9df01 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -6,8 +6,7 @@ # ---------- # mdb admin simple checks - -test: mdb_admin +test: test_setup # run tablespace by itself, and first, because it forces a checkpoint; # we'd prefer not to have checkpoints later in the tests because that diff --git a/src/test/regress/sql/mdb_admin.sql b/src/test/regress/sql/mdb_admin.sql index 65e294769ee..b6b048e5692 100644 --- a/src/test/regress/sql/mdb_admin.sql +++ b/src/test/regress/sql/mdb_admin.sql @@ -1,7 +1,6 @@ CREATE ROLE regress_mdb_admin_user1; CREATE ROLE regress_mdb_admin_user2; CREATE ROLE regress_mdb_admin_user3; -CREATE ROLE mdb_admin; CREATE ROLE regress_superuser WITH SUPERUSER; @@ -32,7 +31,7 @@ ALTER TABLE regress_mdb_admin_table OWNER TO regress_mdb_admin_user3; ALTER SCHEMA regress_mdb_admin_schema OWNER TO regress_mdb_admin_user3; --- mdb admin fails to transfer ownership to superusers and system roles +-- mdb admin fails to transfer ownership to superusers and particular system roles ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO regress_superuser; ALTER VIEW regress_mdb_admin_view OWNER TO regress_superuser; @@ -58,6 +57,18 @@ ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_se ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_server_files; ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_server_files; +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_write_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_write_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_write_all_data; + +ALTER FUNCTION regress_mdb_admin_add (integer, integer) OWNER TO pg_read_all_data; +ALTER VIEW regress_mdb_admin_view OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_schema.regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER TABLE regress_mdb_admin_table OWNER TO pg_read_all_data; +ALTER SCHEMA regress_mdb_admin_schema OWNER TO pg_read_all_data; + -- end tests RESET SESSION AUTHORIZATION; @@ -73,4 +84,4 @@ DROP SCHEMA regress_mdb_admin_schema; DROP ROLE regress_mdb_admin_user1; DROP ROLE regress_mdb_admin_user2; DROP ROLE regress_mdb_admin_user3; -DROP ROLE mdb_admin; +DROP ROLE regress_superuser; From 7816667e5dd7ab5ca247df7d9ec1df0af3b79429 Mon Sep 17 00:00:00 2001 From: reshke Date: Mon, 2 Feb 2026 13:54:13 +0500 Subject: [PATCH 005/167] Yezzey test (#8) This import old CI job from open-gpdb/gpdb + yezzey. This merely checks that yezzey and cloudberry compiles together with no issues. As discussed privately we will redesign it later in another PR. Right now we copy existing code from open-gpdb in order to make sure all out future PR is Ok --- .github/workflows/yezzey-test.yml | 21 ++++ docker/yezzey/.dockerignore | 2 + docker/yezzey/Dockerfile | 131 +++++++++++++++++++++++ docker/yezzey/docker-compose.yaml | 45 ++++++++ yezzey_test/.minio/certs/CAs/ca.cert.pem | 29 +++++ yezzey_test/.minio/certs/CAs/ca.cert.srl | 1 + yezzey_test/.minio/certs/CAs/ca.key | 51 +++++++++ yezzey_test/.minio/certs/private.key | 51 +++++++++ yezzey_test/.minio/certs/public.crt | 29 +++++ yezzey_test/.minio/certs/server.csr | 27 +++++ yezzey_test/generate_ssh_key.sh | 6 ++ yezzey_test/import_gpg_keys.sh | 5 + yezzey_test/install-wal-g.sh | 20 ++++ yezzey_test/install_yproxy.sh | 20 ++++ yezzey_test/priv.gpg | 105 ++++++++++++++++++ yezzey_test/pub.gpg | 52 +++++++++ yezzey_test/run_tests.sh | 75 +++++++++++++ yezzey_test/wal-g-conf.yaml | 12 +++ yezzey_test/yproxy.conf | 25 +++++ 19 files changed, 707 insertions(+) create mode 100644 .github/workflows/yezzey-test.yml create mode 100644 docker/yezzey/.dockerignore create mode 100644 docker/yezzey/Dockerfile create mode 100644 docker/yezzey/docker-compose.yaml create mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.pem create mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.srl create mode 100755 yezzey_test/.minio/certs/CAs/ca.key create mode 100755 yezzey_test/.minio/certs/private.key create mode 100755 yezzey_test/.minio/certs/public.crt create mode 100755 yezzey_test/.minio/certs/server.csr create mode 100755 yezzey_test/generate_ssh_key.sh create mode 100755 yezzey_test/import_gpg_keys.sh create mode 100755 yezzey_test/install-wal-g.sh create mode 100755 yezzey_test/install_yproxy.sh create mode 100644 yezzey_test/priv.gpg create mode 100644 yezzey_test/pub.gpg create mode 100755 yezzey_test/run_tests.sh create mode 100644 yezzey_test/wal-g-conf.yaml create mode 100644 yezzey_test/yproxy.conf diff --git a/.github/workflows/yezzey-test.yml b/.github/workflows/yezzey-test.yml new file mode 100644 index 00000000000..4922f6eca84 --- /dev/null +++ b/.github/workflows/yezzey-test.yml @@ -0,0 +1,21 @@ +name: Yezzey testing + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + build_and_run_yezzey: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Test Yezzey + run: docker compose -f docker/yezzey/docker-compose.yaml run --build --remove-orphans yezzey + + diff --git a/docker/yezzey/.dockerignore b/docker/yezzey/.dockerignore new file mode 100644 index 00000000000..4e92238e609 --- /dev/null +++ b/docker/yezzey/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +docker-compose.yaml \ No newline at end of file diff --git a/docker/yezzey/Dockerfile b/docker/yezzey/Dockerfile new file mode 100644 index 00000000000..0b84f13d7bc --- /dev/null +++ b/docker/yezzey/Dockerfile @@ -0,0 +1,131 @@ +FROM ubuntu:focal + +ARG accessKeyId +ARG secretAccessKey +ARG bucketName +ARG s3endpoint +ARG yezzeyRef + +ENV YEZZEY_REF=${yezzeyRef:-v1.8_opengpdb} + +ENV AWS_ACCESS_KEY_ID=${accessKeyId} +ENV AWS_SECRET_ACCESS_KEY=${secretAccessKey} +ENV S3_BUCKET=${bucketName} +ENV WALG_S3_PREFIX=s3://${bucketName}/yezzey-test-files +ENV S3_ENDPOINT=${s3endpoint} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +RUN useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin + +RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/London > /etc/timezone \ +&& apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ + build-essential libssl-dev gnupg devscripts \ + openssl libssl-dev debhelper debootstrap \ + make equivs bison ca-certificates-java ca-certificates \ + cmake curl cgroup-tools flex gcc-8 g++-8 g++-8-multilib \ + git krb5-multidev libapr1-dev libbz2-dev libcurl4-gnutls-dev \ + libevent-dev libkrb5-dev libldap2-dev libperl-dev libreadline6-dev \ + libssl-dev libxml2-dev libyaml-dev libzstd-dev libaprutil1-dev \ + libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap-2.4-2 libssl1.1 \ + ninja-build python-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ + libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ + openssh-client openssh-server + +COPY yezzey_test/install_yproxy.sh /home/gpadmin + +RUN ["/home/gpadmin/install_yproxy.sh"] + +RUN apt-get install -y locales \ +&& locale-gen "en_US.UTF-8" \ +&& update-locale LC_ALL="en_US.UTF-8" + +RUN echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers + +USER gpadmin +WORKDIR /home/gpadmin + +COPY yezzey_test/import_gpg_keys.sh /home/gpadmin/ +COPY yezzey_test/priv.gpg /home/gpadmin/yezzey_test/priv.gpg +COPY yezzey_test/pub.gpg /home/gpadmin/yezzey_test/pub.gpg + +RUN ["/home/gpadmin/import_gpg_keys.sh"] + +COPY yezzey_test/generate_ssh_key.sh /home/gpadmin/ + +RUN ["/home/gpadmin/generate_ssh_key.sh"] + + +RUN cd /tmp/ \ +&& git clone https://github.com/boundary/sigar.git \ +&& cd ./sigar/ \ +&& mkdir build && cd build && cmake .. && make \ +&& sudo make install + +COPY . /home/gpadmin + +RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + bison \ + ccache \ + cmake \ + curl \ + flex \ + git-core \ + gcc \ + g++ \ + inetutils-ping \ + krb5-kdc \ + krb5-admin-server \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libkrb5-dev \ + libpam-dev \ + libperl-dev \ + libreadline-dev \ + libssl-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + locales \ + net-tools \ + ninja-build \ + openssh-client \ + openssh-server \ + openssl \ + python3-dev \ + python3-pip \ + python3-psutil \ + python3-pygresql \ + python-yaml \ + zlib1g-dev \ + rsync \ +&& sudo apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev + +RUN sudo mkdir /usr/local/gpdb \ +&& sudo chown gpadmin:root /usr/local/gpdb + +RUN sudo chown -R gpadmin:root /home/gpadmin \ +&& git status + +RUN git submodule update --init +RUN rm -fr gpcontrib/yezzey + +# Fetch latest yezzey version +RUN git clone https://github.com/open-gpdb/yezzey.git gpcontrib/yezzey && cd gpcontrib/yezzey && git fetch origin $YEZZEY_REF:test_branch && git checkout test_branch && cd /home/gpadmin +RUN sed -i '/^trusted/d' gpcontrib/yezzey/yezzey.control +RUN ./configure --with-perl --with-python --with-libxml --disable-orca --prefix=/usr/local/gpdb \ +--enable-depend --enable-cassert --enable-debug --without-mdblocales --without-zstd CFLAGS='-fno-omit-frame-pointer -Wno-implicit-fallthrough -O3 -pthread' +RUN make -j8 && make -j8 install && make -C gpcontrib/yezzey -j8 install + + +RUN echo ${s3endpoint} + +RUN sed -i "s/\$AWS_ACCESS_KEY_ID/${accessKeyId}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$AWS_SECRET_ACCESS_KEY/${secretAccessKey}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$AWS_ENDPOINT/${s3endpoint}/g" yezzey_test/yproxy.conf \ +&& sed -i "s/\$WALG_S3_PREFIX/${bucketName}\/yezzey-test-files/g" yezzey_test/yproxy.conf && cp yezzey_test/yproxy.conf /tmp/yproxy.yaml + +ENTRYPOINT ["./yezzey_test/run_tests.sh"] diff --git a/docker/yezzey/docker-compose.yaml b/docker/yezzey/docker-compose.yaml new file mode 100644 index 00000000000..2562ebeaa0b --- /dev/null +++ b/docker/yezzey/docker-compose.yaml @@ -0,0 +1,45 @@ +services: + minio: + image: quay.io/minio/minio + command: server --console-address ":9001" /data + expose: + - "9000" + - "9001" + environment: + MINIO_ROOT_USER: some_key + MINIO_ROOT_PASSWORD: some_key + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + hostname: minio + + setup-minio: + image: quay.io/minio/mc + depends_on: + minio: + condition: service_healthy + entrypoint: | + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 some_key some_key + /usr/bin/mc mb myminio/gpyezzey + /usr/bin/mc mb myminio/gpyezzey2 + /usr/bin/mc mb myminio/gpyezzey3 + " + + yezzey: + image: yezzey + build: + context: ../.. + dockerfile: docker/yezzey/Dockerfile + args: + accessKeyId: some_key + secretAccessKey: some_key + bucketName: gpyezzey + s3endpoint: "http:\\/\\/minio:9000" + depends_on: + minio: + condition: service_healthy + setup-minio: + condition: service_completed_successfully diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.pem b/yezzey_test/.minio/certs/CAs/ca.cert.pem new file mode 100755 index 00000000000..e9f2f1d73ee --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.cert.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE/TCCAuWgAwIBAgIUU9e6chP84r3iZk3JtvnWb1V2N1YwDQYJKoZIhvcNAQEL +BQAwDTELMAkGA1UEBhMCUlUwIBcNMjMwMzEwMDgzNTUzWhgPMzAyMjA3MTEwODM1 +NTNaMA0xCzAJBgNVBAYTAlJVMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC +AgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wDvzjMPE7KmONU +8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwfFL2F5mKHgeHQ +n3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb0JOZis82d+lh +kcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6KNTm42mubVlX +kBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW7aloU5zPBhYf +BaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXvEUPwK7BZzgZy +MerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujDv9l6/T3sCmRv +tWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqubkdJr1Ro8yI5J +m0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRDajZaZ55DopmF ++oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh9UvDhjKS43bQ +zSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEAAaNTMFEwHQYD +VR0OBBYEFJGDr6xmoKJFU6cgS90aFg6lUGbhMB8GA1UdIwQYMBaAFJGDr6xmoKJF +U6cgS90aFg6lUGbhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +ACj87ymjBlgY9UZTUbudHREPPXfqMi2TgWt5hygQSiTrNeQOodnq+Swp86qX/y8w +xtnvc+iILfFnh9ZevHKmLx+JziN4kD4ywEpHW7zS7c3+2QEjIZUwj5qlIg0ByOBd +0M/kpimmuTwlDylBaY12GcFlZcsbuezzm4hU+0qoCV/zi2DvSdAPKXMAeZ3lOkde +PUYJUpRz/QkkxEhSdM3BQYI51mUiltCHMhe6COoN4MHV7tix0Pj9vPjhAVN/4sot +2PgUiCwY8eNQugZhpTosMTSBLZvg/EKG+4slY75/voNTIxWHAHmnPMOAzVgNTya0 +/eP6NB3MCjFuY2E+fGox9YTomjI5oxBr+1LlwVy7wbwXTrgBz9Z4izScAsVbPrk6 +jSrqNeNWK1f+JVnYZkjgPGgPaQVCJ22vdLmkW7U/ATdeedQS3RCApMnb9VCRTUaO +eY4ccuEvj0huhdcUguw6fBjrhPjoPxKMn6S93ginW8Wz9vo8qLkEg2NtQDFu1Omb +cJM5F8uLRr8NotPV5QPg1koHeBv/N2WTRZiUoavAogR9XdyOtrB8+MBu1nsp4Goi +7/suv9XzMJ7IpgXiQfCM++1x7oooyWWdeFTCzqNDJ1IbQDeOCc9cQgeOAPWcIqWO +nAWt08+eToI1YUvjl6UT0bpVaJEACv+/HfBr1T26u4Jh +-----END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.srl b/yezzey_test/.minio/certs/CAs/ca.cert.srl new file mode 100755 index 00000000000..977dab4a3e7 --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.cert.srl @@ -0,0 +1 @@ +53DAAE29A25C5BE967A9E9631F0572E17AC92211 diff --git a/yezzey_test/.minio/certs/CAs/ca.key b/yezzey_test/.minio/certs/CAs/ca.key new file mode 100755 index 00000000000..a0fc44ee422 --- /dev/null +++ b/yezzey_test/.minio/certs/CAs/ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wD +vzjMPE7KmONU8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwf +FL2F5mKHgeHQn3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb +0JOZis82d+lhkcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6 +KNTm42mubVlXkBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW +7aloU5zPBhYfBaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXv +EUPwK7BZzgZyMerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujD +v9l6/T3sCmRvtWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqub +kdJr1Ro8yI5Jm0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRD +ajZaZ55DopmF+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh +9UvDhjKS43bQzSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEA +AQKCAgAgemC4RTDE00J2FfMWublGWmQ991i1kFhdh0Mr22ei40ZIXOY42W/+/15E +V5kcDMiP4/uGtobmVgHzLIx8skK1I6SOuScN6i/hZQBiS3zPC1OjxNfs3GR2y8iD +yzstl6SWriNRShKcBFlBfCvkF27FK1PIz+GpI9xflUS1iXa4nvV/EZrRGgJ7GKPb +pnvwZORGr2In1O76V0iZ8bk4ljo0WHyUcToIFeOSMJjtRrkSWnj1BtuhRP1F/a0O +/VC5mF8w3Zai2YulqJmccHoLMc+wNBqxCiy6lhd+lVzZ6OtKB0w2+m3cF4PjDX8P +TK2gewa9McE5QmU8B/2aNsd/L+r3eGEvWAF/1vRq6NcrFwigq8uCTtgw9edRlDnm +RvICkfAbrwhNaixWwqBVQHoy53H29TohxGNNKa6TTKeJvYEdYKgHx55TxkB9X9jc +iSisqb3fgEl4Yh1Izpu+6nULOqdlldfkKPgKJqVB1AT/avR8J09zmMvW5fPa6fFx +alZ1iVahR5bIFEu1lXygsrBP6N+K/ogyztg7ZKLTIN/FguwMKnXMaUbN/Y/ZZXV1 +oGil9vHKnDrRnUGfcm9tyH2Ddcy6RDoDz+O4cYgMGxDhHran2cicVY1q+Yi08q5h +Napk1phNra5HIHnNHwMxQ75ZKZZ3TOGJL+HMF4yRDj19C/6sAQKCAQEA8a9ZQhWw +0vhZENmSYZgGZLa7RZLSbBzQOX/cetdI6/kvmZVcMvNz4q0/UI9XLkqokL1wJiku +O0zXkaVrBVAsgozp4I3oFqwtcAAGw0KwF4FDAS36k4gkE4SmIUl2eI0XMZCPQIKp +3TB81+XdBITtwfPl5yG+IZDkXNu16qUHEhnhvs/kKhMr8flhFC1J4gdrrQhfuRHY +Jv8e1RLJzMhu/ErRjh82LkzB6m3jp0YxBeIA+9Kkw+OX6SlzRbJPirKxJTaZnB8o +wQmzOy1kTRG4qjKswjdTbzf6549721i8QHwSpwPI3NZQhlSkfsvZ5QL4qPW0nRta +m76YeLlS12yQSQKCAQEAzAQz6OcE6yS2q5UfTZluGaU54Zkm0YSnS394pitJpHoh +JSZlvkL1DzpacquDxa3uQLDikai5TqpNnkuufeMJf7I2ygg4n/v4OFaE+/qj5uNA +3QnL3BVT9DCJ0JvQ1qA5Q/6P5WpUHYB7JHBM9BpaE8e4xocJyWSdcSJDaEXns4Hx +WzhpBdVpPSamqB0VHYg1bv6OGFPfwUaRafWhNzljtxbY8RYcz7IfPmnLImFePTtZ +AjzIoAwUIRFzvmoduda0kQKogRVoEeaW1q6ebPUjYjIZvohnpe27EvgCiTNkcaSf +C96uIxHrSvI8114z9CBXer60xQ0Kz+ds18LtY6w8gQKCAQEAkP/JxlsrHje/f9t4 +9jJ2S4BSNLiUpCZZStYKWmzFJEX5J+SzTyI+uZWFcfi9rlk+brApE8wLH6rHfmtH +HQXv3ldajc21m7yq+hIZ/JYK/d8gaxnBxzebpVYlMb1YZZUIgEUhnOuHq9vGWuVe +x7JUztNccGIPJyY9y/RJXUCrUFHU3Vzun8umxuL+OlO9iu02zbZDb85j52mSfvVp +uwHZjGX6+ZCCOh71DIfnWFlFWikwu+Sx05C9eDbVINCM5kK1AwWR/Ve4ZLBEJtHh +5lcmen4ypcb5uLVWRA0SmxPOxcVqj2c24D94Sk+H7UayMLKqqvvW45cgsmYUJgHR +0MsieQKCAQB9goBk4erWtmliuYTeemuPf2RSc6O79b3t5mfU4oCVnUTS1AJ3wD1+ +tsl6DiYs8MnIJoncTk5iJMdHgQvCCnCHjJ3EQLaFRb/4+NErK5C1tEztLt+pb72M +VmgSXCloQH26ZNslqfpBhA895ZCSA7wyuwXjrKPKsAlj1k5d0dOvTVusYNHLcvUh +V6vjdLDO0EL/G79THBZlkwJWi3Q4wyejNX0VJCNpaw1pmjAL4JbXWLFzfO13+LZR +eakZFbNf5sSDCX2cnAzAJnnZbOet5El2WZgY7VXGcLBMBSOaQHGksD/gT4gVrypv +mwLvA9c2cscejkArkdB7AsalHhho30cBAoIBAFJBO0RU7o0S+F6KHIP5aFbItcUd +NfUgoJTAFUD3EnBirvDv0pu8T8zkgKf7PRFkZQIOXocvpX0Zy6N7fiPbvzTA/vH3 +mFqias89pTUAgv43R8ZsAC/qlozUuByegigEz2zeVd34w7MdkgGo1jnqmijAIXZE +INBo0swkxAbix+W1Pur/yvGUpC6xu3ISmdrn0p20B7QhyuoqC3ea/az7ePwx+Pu9 +Jl8tzMujbHNHhw+OQAQOPHi6EUPs/H37euj3G7oBaVUwXJq3Tbwg95W5Jih+CgTB +Sbe6eYpR/j/SYGwbS6/DbHi3IjvblN+2pSPI05JvXMhLC/lAeqcdVJAgTvw= +-----END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/private.key b/yezzey_test/.minio/certs/private.key new file mode 100755 index 00000000000..9d12f84dfa7 --- /dev/null +++ b/yezzey_test/.minio/certs/private.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/ +mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm0 +7Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa66b88Dz/mHDK +JTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6ANfTXmhVuVm4X +7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOzGoAY1iJbHuXV +tzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQdGbdmKiwu/NL +oc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxG +ScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U51xDRk6qkQbO +Ah5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1co+0Svm9EEG9f +bZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+ +flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69fMGozzkCAwEA +AQKCAgEAsWK7PvzUcBzosK6GW6/HloJCLniOpyOS50LTisfEnZ4qGn9lElrwv1X7 +bliaXsutz+rFbHdVQVE6fhU723DtlAhaUS2WC5n83j5aP0Lv93qaQIkSLj+DoQuk +UGIWetQQoPFG1gEjXoAV1k9tsFiXTGz8RpnDmNb2PMW1u1AF1G/gygh5Ape0fs8C +YwvMCnfL/eEqGRY8D8526+09YGv9ijXle32MLLHDuHWdfz0aPayzHIZIvXf2Unrr +vUwJAZ/ONz+Obj0etVgDpDrDD5SCWVer/Jlj/xT2Dfg0W0NBYkENHpJRJwyQNYJu +xWe7SIKLXslY+JWavhhf3nRkjOJWIGtgix6Sbb6K3TFNu65t5nOVr9dyZFcW4sSD +JpyUjDK6mbpUKgh3yaU9QXQSY0bD48mTt1UB2vmh/dA9OIRRUXr+V4MKPQBtJt0V +7ay2P95oWSHM+zpzPw2Bv6R6s78kaFisD/IqVwt95NMSiVTb3rNsJLs+KNKjuoV0 +EPeZuiQvDZDNi2+pKC+3tT65+sECno6ZSZdvn7naRYy3X+QWbOY/zG9mXIMyRXSs +oDXPH17838swDO7YrmWDvHO2Coz72eEftsovDql+D+4w6lq5DZygbHiSO3yyB4cR +AbI1hQ0nneQC8Gi+YPNEguAz5Sdvys91urZ1awllsTpG6gFPuAECggEBAPROYIyi +4mjr3MAUU0h6FHQ8uCARknM2NOZGDpyHmOtFjdTW47xGB3E8Dhyci0iOYjgwhcNe +lx/gZySKuCbC4HbxWUzLFYOOYRpj41oIZW3kvnkc7vVEst726CO+cVtTc8cjYXQL +pFQO3wN9C+S7OXIZOSO/P83jHm3bG9vQ1Kgh7gMBgZkwx3NMt2d1slEo4NViXZ5n +1960wWer5J6lQtpgwJYik7tZZBXkGA8QrhvzqfAGLYUQc3chWSHqlRt/+YJzFXZc +JYhpmCOs3jefC1I0T4wsxTJAhv3xnlGY5FDFIsHXqkCgwsqU9rYNfNQLLhuuwG3t +kR/sZp1eKmkDCjECggEBAN+Z1DqZIgd9rXTeUICbzlQy4891BhCWy3F20CTbzdfv +7EN49uhaD/OltJ8LJcDqm2F38Xjz4+svGfsYkqe0EMV1IOkdhnf29uGcWb/kWGGX +FtNRL3QhKntouVqqsJdeFNcvbPF4RZPUOQTiHjH2U+nTA/KIKU/nxSqJwR3fj9nc +v05k2jv+eoodDx4Fs/zS/cYzA0bjEXZlO6fS+MSWJwQGVbd3lqieo0FVuD3Y2RVs +nQidKUOm/qTE1/r//ggr0nX/GD6n2gRyUTV5yHIoZ/ENCOuxu38qyOk5ko8knPeo +IGqluaaTCyFav72vS9IbWVUKicdmzaLaYVLG1EzMS4kCggEAbHhoMckYUZF3f+kG +WUWq0zkqX0KuDW1h62PrlOA3qy5EnN2UW8GUCFirw1RWGy7suRoCKg5TdxnBcd4N +iVg5JVZfWdNJiBGtV3RGO3FC55oKX+fSyR9pc8mYpFYoKm5RF3fECywoGBJKPlhE +8ZeXF+vPDOobQCS5G/mO0vhTkxsNfFQs4IFh7PWA8PS26YYG3XdBLdXZaM5EmO0f +Irkph6AxofdUNHiqEXYmoHm9dDepmiKED3KQcbJiysZG0eDuPVcvjk+3cmu/2wcj +2vpIooULVKeEHp270gB3VK9Xn+0HU52Xv/4gvqWJLKesZtFX9X6Pfb9fEhd6MCDh +H6Lz4QKCAQBVTk/cp02HRBhoDOTzm/2ku+GT5VaR/6XQPP7+AOnQZ1VhDIZOcQXC +88YsI0mdBySOk/8JISskL80esepJlvYLDzumFECYBh18R3UqM4jQep8XsdKD4J9f +g72rbJGAvkD/M7XBjhwlYQL77PSOJScwfvPzlKUGOitLplCKAB/Wg1RtKSblWpoP +lTSORKi9XRW6hv3KDpk77TVMOr3z6kEqVOxg2XweCp/ILlbjKZbwiIaEocj7aXuf +CwZHWWDmZymdINx0Ev+nUKHQ4AxsGSRGn612llEtG/qYmAFlIRfNmFP9vApPFaxa +Zk/eo0EvAIYdXq7f50Wuytf0h6y5O/vZAoIBADrF4jHV+kbwWso1Rj4SA9xjmDSS +8v99Cr8YReoEwJoRo7sLa4XssY+deIvvR8OUwHmbPWjzDXeJFSMvJd9H7hytPvTW +wEPXd4eTRwcVo4d2ZUxszmJyLp5Mx/0qMtI2XkYPRfulPhXReYjkezjJ1mIzQiwL +RS4vwafBs1LvmbAM9stJt1K/XoI0e4kTYd9KyruKRW+JeuCrxcSV+O4nMRyOiCIQ +cHFXSlSHDyQo4z46rp2+IhIt9/Vq5DQ8hhbAtIjqa/ndFUj8FaQs6sOQq4410+LK +OCT7UGhKdY9I6fV/O4MAiLzyXcXNo6SRZ0YmPfam2v3gCqdDHLneLsbMoJ8= +-----END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/public.crt b/yezzey_test/.minio/certs/public.crt new file mode 100755 index 00000000000..4da1db77d25 --- /dev/null +++ b/yezzey_test/.minio/certs/public.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE6jCCAtICFFParimiXFvpZ6npYx8FcuF6ySIRMA0GCSqGSIb3DQEBCwUAMA0x +CzAJBgNVBAYTAlJVMB4XDTIzMDgxMTA5MzA1MFoXDTI0MDgxMDA5MzA1MFowVjEL +MAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3QxDTALBgNV +BAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMMAnMzMIICIjANBgkqhkiG +9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ ++Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzA +KG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa +66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6A +NfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOz +GoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQ +dGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7 +IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U +51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1c +o+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJ +yHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69 +fMGozzkCAwEAATANBgkqhkiG9w0BAQsFAAOCAgEAipaTRZxvDImsu/pwxSHEkKFz +ndKt84oEeuWRBo+sT5wZAcUJicz0tHVULwhXiqwMRC06LBqFxGzI/srokK2QiN5B +eikQvWEsr08tZuVm1ewNit6JzlwbaLOJc4DSTgJtWXKjWaIzqWsFl36ViLO55yTb +NgMgmpDhBl6hQ7yKtaSVJ+xiMKSd9nz6gRmJMz7sLchFmy3fTYtayUJcaDsFjEm6 +yTYM1oWj05xVcZJtASB8Bcp+XgeOdNbwvvTGrM2ctDC1GkPCK4X63GJfAuCv6lnV +ggzr2Z8dg/YVLCbmyvJq4n+rwNbVDebQBzRzyU9id0dRccyfPQzB69LPYcrMFzd7 +h97i5Vk7Ar66VeKxdw3Lbyl91yTUy7e7EdYkBrlPzMohRcuCEsFPaUdOoiy3dmUX +PeaubgtJpaNlVbp08rdGAgcrY4aqmvVM+tlKWfgiEnVmQ4vKTcrITnhintvttXyh +GrddsvKAE0m1rDpfd9BqXH4FEydFHoL7oMpKHnu9LUQsPHnwGvpq75KUN4j2nP4P +NIEmmqOr5SY9zcp9HQApxWPscQckb9aCIDZ8MMrceNSxRtU+bw0xNs+IILGx6dNA +cK+bDLPtMneno6XXDqqiitE2ohXf5WIeB3kPLOsp3awVhr/g+pqjMSSjyUEC+KXr +dkqHW3oKT52eCNzR350= +-----END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/server.csr b/yezzey_test/.minio/certs/server.csr new file mode 100755 index 00000000000..c223c34efb9 --- /dev/null +++ b/yezzey_test/.minio/certs/server.csr @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNV +BAcMBFRlc3QxDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMM +AnMzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR ++rDY2+IK57DktPou0KlJ+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZH +DTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz +7pZArnIrZdOwixBiu8oa66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKd +MqR99irHKvGtC6TN5o6ANfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhya +oEBw/aQmltfuEilMxrOzGoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zE +YqxVurYQTz0cWQZxo8BQdGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCI +JiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh +9CHfNIytESa1zb1H/d+U51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ec +maGmnAA3SYRZYV6yYf1co+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2L +KJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERU +FAlEAvag0+06qhbVyt69fMGozzkCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCw +gv4AQ9xh+LP16NQ0QCZ7QM8y6cA7jb7yqc8G+gvTjbacsuxpfI7qiJHx/Aon/ReL +D3Lra+HKzQticJaqFQEiXE6kXWB4DndP7hY7MF0R3xlBvNWyS8Q6Nr3m6IQiJx1Q +qd2Fgc7341wOMf3XMW/E1XUEsnzL8pWrKI+lvHAMI3u2KvsqwTtWtFJ4HMQoab7D +Xi4QYVk/DMF/lirPgcm1xnDZP07f7lzIGasO42XOBW1nV33w3bqVjRfKLZnbCs0Q +IOecmfseUBER9ycTUnFHN99BGx1SmTtXmqoeQIwijIvMoBNtEsOskqhW1+snFlNO +st9pyjwhuASvSWDFbEsaC1tl+5oTK31XMrxs99TDRoPMY1UwVFbrXrc+XWP1MSwg +0SB2c2DnLwGT9Lp9w1+epn9oa95B5JskwRp86Lbf5y+XV7W3Vp/vCzQ+db2sUHuT +bEUpNJl/KcxxseMMkpIJy83w17RhlRsgMKvAQtftYWX/z/RHyGKWHHPPoxEJaKyc +hm+X65QJd8QoOd/IXnkCMTQZocpQxoKBbYqdTBDxYfXnTILWHRUb0OfmjS9fEmBE +UqYdwNawOIXKlPsI/JseO132C3TtxjmkqMSNixTFyRvpoqy+/wFul3QA+yF2eCM0 +pfKcr8QsHn8WEVqjEmyQiM1ixrDIgIDc1jurvLX2cA== +-----END CERTIFICATE REQUEST----- diff --git a/yezzey_test/generate_ssh_key.sh b/yezzey_test/generate_ssh_key.sh new file mode 100755 index 00000000000..1df4cf14b60 --- /dev/null +++ b/yezzey_test/generate_ssh_key.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +ssh-keygen -f ~/.ssh/id_rsa -N '' +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod 600 ~/.ssh/authorized_keys diff --git a/yezzey_test/import_gpg_keys.sh b/yezzey_test/import_gpg_keys.sh new file mode 100755 index 00000000000..fc351a3b416 --- /dev/null +++ b/yezzey_test/import_gpg_keys.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -ex + +gpg --import yezzey_test/pub.gpg +gpg --import yezzey_test/priv.gpg diff --git a/yezzey_test/install-wal-g.sh b/yezzey_test/install-wal-g.sh new file mode 100755 index 00000000000..bfad4e82a58 --- /dev/null +++ b/yezzey_test/install-wal-g.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +# Install latest Go compiler +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install -y golang-go + +# Install lib dependencies +sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + +# Fetch project and build +git clone https://github.com/wal-g/wal-g.git +cd wal-g +make deps +make gp_build +mv main/gp/wal-g /usr/bin/wal-g + +#Check the installation +wal-g --version diff --git a/yezzey_test/install_yproxy.sh b/yezzey_test/install_yproxy.sh new file mode 100755 index 00000000000..9e3abd1d9cd --- /dev/null +++ b/yezzey_test/install_yproxy.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +# Install latest Go compiler +sudo add-apt-repository ppa:longsleep/golang-backports +sudo apt update +sudo apt install -y golang-go + +# Install lib dependencies +sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + +# Fetch project and build +git clone https://github.com/open-gpdb/yproxy.git +cd yproxy +make build + +mv devbin/yproxy /usr/bin/yproxy + +#Check the installation +yproxy --version diff --git a/yezzey_test/priv.gpg b/yezzey_test/priv.gpg new file mode 100644 index 00000000000..1b9727a319c --- /dev/null +++ b/yezzey_test/priv.gpg @@ -0,0 +1,105 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQcYBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb +aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA +0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH +vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP +JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q +ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa +OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt +/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d +w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 ++TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ +HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB +AA/8D1gsTYmHqNYbLNEr9nsCIt6XL2Abg6s51Vj24z/cd6HJOhQEoeDxdWNav6NL +SPh5KDJNsCk4JvnqAlKgdIx6CXLe9XQvCeB9hk3Zae/91GeYFb9sN8GFkppf7ZC2 +TEf10upfjo6USnc9lkEwv/R2Hjyerk2WX+rYldvol2JneF1hLV2OuLyyY4Hm9mQx +qR30f2/u4gXBzFSn19A+hYn999qDsBiUnk4/mZ590tZwjh2mpixxTzS9HhbMLCAL +8w01rhVVqgZTyhhK65aP1cuZCgiPsy/QSDYzkz3F7sRcmBxDyPul85koWbLPk99M +C4bY7xMBKH9m8rRcXdCGiAuBKRlIZFQcyI7Lw4WxqxdJROAjhYb+FXDKSst+cyw8 +c0qP/UbqnNaZmWQ0w95UMlwu8MZ6sJK+CzaLkjt4itkN27/DCfxyfuETqEiTO1ba +yJoVDE5MH0CQLIlByvJAU9AmNd1vLL9CTzm39m8B+eOrPR5n2khu85D+VR3vWZo7 +XnM5RTXhKTMfGFCSunNVlnABEy1eFZYFOogZOkVYdqxRTiZccDIZgV+spN/op4kA +SofCehr1xxdd3DXRar/GOz8J2jrWyqaQzhsD4kKl5nBM3NH7fAPsBBv0VH1DE0OM +ZsiB0yQFyHTWeQOcNla37pD1H1DZa4fpsL6m3BqSK38cKcEIAOUr9faEnjYPTAoW +TKWpoqt2ghyGsklrdYVcgb7pINam1UV0f4uvMBv702ZnLwoC0Rn5ITgPXH0a0MDM +H9yfWZgb6L/mmE+B81D+EDUkb88qafXXq9TMspbwbSuY/DWFTZPVfRDWYacr/IPq +XdtZqNBR2BWDeckgBcnEwg/PBn2HrrpfWab/D9Lt1QFY5baCuFevib7/D1DxO9qq +57iA0IDT+YfydYawwixvarkCNmUfkDowhR7DqHWDppWt5n7rxiTuE7I7C88pV4mV +rVTvXeMh6ooB3m2yW1FhsTgT0DrWxul1glCASQ+TfOks6dcDMvcFZFSzWT1r2rR+ +LqNNpDkIAOa9LcZEaRV86QLLcBYpPnaDAFlZYU/1b6QRbU3L77rULvtHSVegEApp +7a1mDrx1bTp5Z6TGq5ls9bMHF4tg5IcBLDHsO6dYAl2vAbYuCwjfPa3hu/oK+J5F +9IQLrvS2vM2shqhIfIbKglXwAWySLQoR/CsAMgH4vk/R/08AJQRyXndaY3rLImQg +ofUQaRxl4950wO/RJSQ9RD5q1bvseJ6bZNlYu1SoEJ/uOGqlMbMP9DAUb7LnTwlt ++9Mdv/ibYBABnUh5hLLCc5lBgeN+Pd3AAEygvbK6rW9a2CV4vO5U0aBU7Um8WAUz +suVEQM/rMTJanxHAeHgRYMDccadTAZEH/2i/xd20U2Taorj+/Lrq21hQJ1jKpyN+ +CnglPV83pGX1V1j007C2zhO/YB5qTh9ztZvtwkuX5EoUN9mGRZ0t1K3xbmwlDQbB +D1hofRcN0U2h+ZXQ+aID+gswLhw6jWatHiDmhIGSPzopR/1oZrQwS0CJuqb3XQFP +lAgeMPEuFufIf7tijFLL3PDuTPnG5lndI6+uCw+BhkfgN7jAHHREQlolGgzo3mwW +htGr6cOd9W7a9tGN7nHb/yVH22gnXs++wyC8OoeuEgAqQL2CyNOEo3EyN1K79QH0 +7qaKg+XHJy/NymQdcALxp3zUNh3UuOd0CZxPudTVfD5j3cD5fST9Owx+CrQsUm9v +dCBTdXBlcnVzZXIgPHJvb3RAaGFuZGJvb2sud2VzdGFyZXRlLmNvbT6JAk4EEwEK +ADgWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbAwULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRD/Gb3KATgePXGWEACRiwzgx5EVSpQEKe79sRAkO9hmAB33 +D4dN4y+D+2kUY/paPg3etCdPIJQs6qFFu+HIWnCTGiUepcsBeUvRpTDwguP6MtXx +TObmaU20ZoSbZZjqnGZ5V9VlwCQjJWj3RN+7BYVsECPJGfEMW+M2nHX3MrGpohfo +KpacoL2kjVswvWocGHNIMHpVHlyGGUtwVHz3kO5HqwcZvWwZj189dKTMUSsfbXdz +723BDp+m/NLUi8Ki/BZpsRwSXqGbpdQ8LYaSJvtS0DY/rIILIOyIfhnHK74rgCip +0vfZsfypc58YrPrj9+pfF7Rx00m0RfGll+lD1Pha4GStS1PvtpdtRTvf6qsmArON +q1N7ha8YJHtPueTKln/7ik3jqir7vO7SLL+CTa/XfS1jaJhonHORfW+fekndXK34 +zrQ75cFn7D5BqqcTXu2hkTDbPqd9aBorWFU/vbqZRVH1a/RRcVU9SODnIwdCm06x +XLoujcr1daZd9GWT0PMvFHaehQ/4zbdd/pYJ9efU1zlfpdwxUpBWyfUEOGbwpHGv +7FjXIHSkHkTNgaOK5AP0FsSGfjAzTxq/2yLRQKm5f3GqWNOZ7nprLg85Tyxh/gdZ +rk8o3/pW7P/XDLU6jfhWV5YR2mxddaFs6P7D+jvRf3EVHbJHtcK30usIWjGDWyNk +310w6Q3CumfEo50HGARk248xARAA6LnhBgnfn0B405FGDbb+MOj/0dmmb81XwL52 +ZqbAIsomMUfrzzFHJb0PtexSsIr7I+NweVCSvc3UaGcMLF8pqOVnP4pkPXi+hBlG +eNeE4KQvMpn56nLsZ90GtpKYLLsMQB6VMm98+v4wCJ6mPkM89SPQlc7q9QFfC31g +trbb5Y//eDqCAATiVIdZ6nEFNKgNDMKC0UmRY+D3RaeElPpWO5lWDDldSjOZ9/Or +dg/TSdRviK7+NzdyOM8xUc6lQunG26qRH/bqQzHluBYMp4W3l71Ojf8YlSATkziB +dAdwpQ1MVx4rCk0o5nYuGf58sgxTIr6qIR4777Z7lanc9utcLeSbltkid+zATW8x +PFPG1ID6pNZ0JX2ttQ7HQAb3RfNDkZxGijqEnJyjY0nJON0/8Jcz46GkeUzkae0r +I/TOS7OASs13NWMQV8TsqDy9VMT+pcTDjy1Rad8krXtraK93WlIPWhb2vCSJkCMr +412bl2SOrtYUdu093yLYkEkrq6vZsl4zkIPrCbuEdJZgSn5d23nHd2ARsaj6D4KZ +RP9dg7A7RpEezNLAh0FDwMLe/nJpNfQdd9MXITALc0CwJ/x0sFBrj1+q7jELYtSp +zh993C4i9ySOaWKfOGV9UjeSLBYfExuxisMQyAIoD8hhNOHnqWVgcYqJ7R3w0ymU +96M29M8AEQEAAQAP/1WZpSiM8ilH1AlxmlRKFjYURaBAz6S44UmeZLt+IxbIxwKC +Yzxq8jHx1/kAyytve09ohULB/a99qV6bZJFfkVmzw2XON++aXW0GRPMGxrPAADI7 +C38ONWFAnYsC4aE2TZu6BAOwmUZSv4U0IY6uOZorSboIiUiD8BswSyX5nWlTLVLi +JlXudfdEb7C5UIJdO6uRUf+78RPNN/ZxVuVbLOOwE0Pcx7EWyM+4Wz1KNdumnT2n +rA7QQJ2frBLckNHLXh8HHmkk72a20DmFNrNZjj1sXpwBE+AqE7knZAozAF5dRVKX +4Jnh5qTaHDvobKIqwVt6yOX0knQp6UwT1hgmWtk8584hLoxuQbzr5yM3/QNiPMka ++aMGu1COS34ppIQfETt1C3Lv7tGcFXkadyV7JfVl0u8qAJjGzKgmrBcVkDBZdKT6 +rP3QAAwpcwdDaY0rXR6FDrHKZWZhvsSNjX9vXf6ThCG4/Wb07bMia1Y6UdtNWYAB +uDQkFJszng03tQP8P1p1J3XouzOb0XA3tkusOoRnMlR47oDb1TlUJKuJv7ztK3AI +5v7FyJO9TJbwhBWWno171KKGn7R1vciNYxWB1IqGamLQFKajTFKYVikn7mCdfDNe +zVgGcK9DAgoJXtclmjtKzUnqfW6ctJKrNKXkyEgkbWKGv33skwDSMZxakTpFCADt +dzG+RbDlZrwlSMhv+33vjV6arulEY32vj3Thbphb4v1JwOTdXNVwF5pC+pf7sUgx +ZAbEuVix4zu+AznCdRGPYjAHZgUe8ldp7JrQuVzlknBJ0Z+DedXcvL8RAs7rFedo +BO9FTqIolTqFZyBJeaDJcMTwf8qQdE71+n/r9oigKi/8UXSAZInnvgYMTUjgegqx +UyQJTEXRQYzUfCUHFpYqel1NwcsLO4JciDwb0ak1QUvIy0HulLrRKrRYUZ7Xy3U9 +XBjt8BPWKdNfN6qKWK+7tYzPNB2BLQT2Z+l0L+Ir9TaWxJTZdfRysYytAhTl3ZoL +99KlA7qIxOs7+DnuR8jdCAD64/v/xNl02ck3pBnnxjnaspe3xEz5D9s/a9Ob+2MU +pDp5b0ZwPJ0tpk6CguiVhi1gzRXzjABPrCbsBOXk2gQ+Pilk3jkugG+wqg4LOLzy +3AiDbD+GVmTL9gdhPNoNIjPBz1tkZEGdTVsUOWT5Kuom3KvbmzowdlTVb4x/kkRM +oOTfO1Uhna3b/pcCsMiHXkM3ZME+DFEy6Imh38742ljllECCDCUAvd4drbx4VaNU +rXyI+j3NP8j/vHhWCld0Bg/C5Xq9ioDOzyrvApTat2OZF2Z9K4nYYTTwniAHJWNk +hIPDKyIKeE4Y/WBrSfVQbdtTNi0fj+3RTtS1j4uSwMObCADf3miT0aX2pBxy7N6D +krhJwq5Usaeu3fVNOOm/HVQsouqJ0BYPX412YIDtL4iY/YibVdzVlNg2hQuPJYGx +hZF/isY+qpckLvdABbtreBDxoqtqBgdOI0EGXpBuJTq9iuh2kiKFWATEchCMND9p +wZK0jrVtQ7JnXwW4CRBnN0wV2LO8+VF6How9FSdVRykG4cRed7JB5U9B11PK3tay +v82+YnmTEr7YuMiJoXHt7+9nA1ftd/Au0IegwvmRO89MxvpGeS8GVZFdYLP5UQiH +bdrnFmLyQEsPPIM3yvFQCfb2W4WmxH3PycTrNDs7mone1L/pk/eMdg7QYF+E/QZD +1N1pfKeJAjYEGAEKACAWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbDAAK +CRD/Gb3KATgePW6sD/sHl+DskI2dz4Ym7ZKng39QHOKFdzoK9xtnLQSynqfrwzvz +UEzsqWNjOHONfzat9savfGBvWB6ULz3vQ4yZ731Y4Tdc4UmVo+79hIgeArNiLyou +p6tvcl+bKOv72xlrYn43lYy51Aj5TsU6TBGVvRETnmOWvzGzsQrhcZ+IA8p4JXoy +fiAM+reuc1lwWYZWS5n+XotskFS3L6BIXLPW1QIHxvcMkZAI8/VeloQej1z4t50i +7mAAXB/RwjXGCLnhFxhjWLHglrxpg6JpsuCkI7j5csYQuzhKrSWa3mxKv7rysRh3 +VpIkHfRRu/kyfOwM4vYCRsQH2lBl/JVgAn1koZKdnGomoq/Gw7+psuYNGX9+CQ5E +ShHGWGE2TUjCi88FDLDpOOmpKMoK19HbNGkeVeKXY80D5LhjSrOL61IY+3Hzyb0a +JhHA7dXAqMHak38Ps5sUrEtgDBjkhGtctCUGV9OsvlqAXwIDLB96W5LKJKw48osE +vcfquO0z3bMKyr0irCyBzn3sPd2nubFy/o1umF8efg2OqWybQ3PZLyMojQ9uu8Yb +SLM3+Q1vX/yAY8+KmaAvvUuTCS1dRWAM0s1g86AUCZ5LbMa0HwIGjCSA2nUw8BD8 +l3sVHJsSKOhB0XTUBVkIDBPdJcj/K6G8E0PGO19UQEMRis0/jnPPL9t8G63xQg== +=uAYd +-----END PGP PRIVATE KEY BLOCK----- diff --git a/yezzey_test/pub.gpg b/yezzey_test/pub.gpg new file mode 100644 index 00000000000..139122aea2a --- /dev/null +++ b/yezzey_test/pub.gpg @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb +aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA +0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH +vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP +JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q +ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa +OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt +/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d +w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 ++TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ +HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB +tCxSb290IFN1cGVydXNlciA8cm9vdEBoYW5kYm9vay53ZXN0YXJldGUuY29tPokC +TgQTAQoAOBYhBAEddTLhLrHrUjsrUv8ZvcoBOB49BQJk248xAhsDBQsJCAcCBhUK +CQgLAgQWAgMBAh4BAheAAAoJEP8ZvcoBOB49cZYQAJGLDODHkRVKlAQp7v2xECQ7 +2GYAHfcPh03jL4P7aRRj+lo+Dd60J08glCzqoUW74chacJMaJR6lywF5S9GlMPCC +4/oy1fFM5uZpTbRmhJtlmOqcZnlX1WXAJCMlaPdE37sFhWwQI8kZ8Qxb4zacdfcy +samiF+gqlpygvaSNWzC9ahwYc0gwelUeXIYZS3BUfPeQ7kerBxm9bBmPXz10pMxR +Kx9td3PvbcEOn6b80tSLwqL8FmmxHBJeoZul1DwthpIm+1LQNj+sggsg7Ih+Gccr +viuAKKnS99mx/Klznxis+uP36l8XtHHTSbRF8aWX6UPU+FrgZK1LU++2l21FO9/q +qyYCs42rU3uFrxgke0+55MqWf/uKTeOqKvu87tIsv4JNr9d9LWNomGicc5F9b596 +Sd1crfjOtDvlwWfsPkGqpxNe7aGRMNs+p31oGitYVT+9uplFUfVr9FFxVT1I4Ocj +B0KbTrFcui6NyvV1pl30ZZPQ8y8Udp6FD/jNt13+lgn159TXOV+l3DFSkFbJ9QQ4 +ZvCkca/sWNcgdKQeRM2Bo4rkA/QWxIZ+MDNPGr/bItFAqbl/capY05nuemsuDzlP +LGH+B1muTyjf+lbs/9cMtTqN+FZXlhHabF11oWzo/sP6O9F/cRUdske1wrfS6wha +MYNbI2TfXTDpDcK6Z8SjuQINBGTbjzEBEADoueEGCd+fQHjTkUYNtv4w6P/R2aZv +zVfAvnZmpsAiyiYxR+vPMUclvQ+17FKwivsj43B5UJK9zdRoZwwsXymo5Wc/imQ9 +eL6EGUZ414TgpC8ymfnqcuxn3Qa2kpgsuwxAHpUyb3z6/jAInqY+Qzz1I9CVzur1 +AV8LfWC2ttvlj/94OoIABOJUh1nqcQU0qA0MwoLRSZFj4PdFp4SU+lY7mVYMOV1K +M5n386t2D9NJ1G+Irv43N3I4zzFRzqVC6cbbqpEf9upDMeW4FgynhbeXvU6N/xiV +IBOTOIF0B3ClDUxXHisKTSjmdi4Z/nyyDFMivqohHjvvtnuVqdz261wt5JuW2SJ3 +7MBNbzE8U8bUgPqk1nQlfa21DsdABvdF80ORnEaKOoScnKNjSck43T/wlzPjoaR5 +TORp7Ssj9M5Ls4BKzXc1YxBXxOyoPL1UxP6lxMOPLVFp3ySte2tor3daUg9aFva8 +JImQIyvjXZuXZI6u1hR27T3fItiQSSurq9myXjOQg+sJu4R0lmBKfl3becd3YBGx +qPoPgplE/12DsDtGkR7M0sCHQUPAwt7+cmk19B130xchMAtzQLAn/HSwUGuPX6ru +MQti1KnOH33cLiL3JI5pYp84ZX1SN5IsFh8TG7GKwxDIAigPyGE04eepZWBxiont +HfDTKZT3ozb0zwARAQABiQI2BBgBCgAgFiEEAR11MuEusetSOytS/xm9ygE4Hj0F +AmTbjzECGwwACgkQ/xm9ygE4Hj1urA/7B5fg7JCNnc+GJu2Sp4N/UBzihXc6Cvcb +Zy0Esp6n68M781BM7KljYzhzjX82rfbGr3xgb1gelC8970OMme99WOE3XOFJlaPu +/YSIHgKzYi8qLqerb3Jfmyjr+9sZa2J+N5WMudQI+U7FOkwRlb0RE55jlr8xs7EK +4XGfiAPKeCV6Mn4gDPq3rnNZcFmGVkuZ/l6LbJBUty+gSFyz1tUCB8b3DJGQCPP1 +XpaEHo9c+LedIu5gAFwf0cI1xgi54RcYY1ix4Ja8aYOiabLgpCO4+XLGELs4Sq0l +mt5sSr+68rEYd1aSJB30Ubv5MnzsDOL2AkbEB9pQZfyVYAJ9ZKGSnZxqJqKvxsO/ +qbLmDRl/fgkOREoRxlhhNk1IwovPBQyw6TjpqSjKCtfR2zRpHlXil2PNA+S4Y0qz +i+tSGPtx88m9GiYRwO3VwKjB2pN/D7ObFKxLYAwY5IRrXLQlBlfTrL5agF8CAywf +eluSyiSsOPKLBL3H6rjtM92zCsq9Iqwsgc597D3dp7mxcv6NbphfHn4Njqlsm0Nz +2S8jKI0PbrvGG0izN/kNb1/8gGPPipmgL71LkwktXUVgDNLNYPOgFAmeS2zGtB8C +BowkgNp1MPAQ/Jd7FRybEijoQdF01AVZCAwT3SXI/yuhvBNDxjtfVEBDEYrNP45z +zy/bfBut8UI= +=x2ib +-----END PGP PUBLIC KEY BLOCK----- diff --git a/yezzey_test/run_tests.sh b/yezzey_test/run_tests.sh new file mode 100755 index 00000000000..68dff441c49 --- /dev/null +++ b/yezzey_test/run_tests.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -ex + +eval "$(ssh-agent -s)" +ssh-add ~/.ssh/id_rsa +sudo service ssh start +ssh -o StrictHostKeyChecking=no gpadmin@$(hostname) "echo 'Hello world'" + +sudo bash -c 'cat >> /etc/ld.so.conf <<-EOF +/usr/local/lib + +EOF' +sudo ldconfig + +sudo bash -c 'cat >> /etc/sysctl.conf <<-EOF +kernel.shmmax = 500000000 +kernel.shmmni = 4096 +kernel.shmall = 4000000000 +kernel.sem = 500 1024000 200 4096 +kernel.sysrq = 1 +kernel.core_uses_pid = 1 +kernel.msgmnb = 65536 +kernel.msgmax = 65536 +kernel.msgmni = 2048 +net.ipv4.tcp_syncookies = 1 +net.ipv4.ip_forward = 0 +net.ipv4.conf.default.accept_source_route = 0 +net.ipv4.tcp_tw_recycle = 1 +net.ipv4.tcp_max_syn_backlog = 4096 +net.ipv4.conf.all.arp_filter = 1 +net.ipv4.ip_local_port_range = 1025 65535 +net.core.netdev_max_backlog = 10000 +net.core.rmem_max = 2097152 +net.core.wmem_max = 2097152 +vm.overcommit_memory = 2 + +EOF' + +sudo bash -c 'cat >> /etc/security/limits.conf <<-EOF +* soft nofile 65536 +* hard nofile 65536 +* soft nproc 131072 +* hard nproc 131072 + +EOF' + +export GPHOME=/usr/local/gpdb +source $GPHOME/cloudberry-env.sh +ulimit -n 65536 +make destroy-demo-cluster && make create-demo-cluster +export USER=gpadmin +source gpAux/gpdemo/gpdemo-env.sh + +gpconfig -c shared_preload_libraries -v yezzey + +gpstop -a -i && gpstart -a + +createdb $USER + + +gpconfig -c yezzey.yproxy_socket -v "'/tmp/yproxy.sock'" +psql -c "ALTER SYSTEM SET yezzey.use_gpg_crypto TO false" +gpconfig -c yezzey.use_otm_feature -v "true" +gpconfig -c yezzey.use_gpg_crypto -v "false" + +gpstop -a -i && gpstart -a + +#run yproxy in daemon mode +/usr/bin/yproxy -c /tmp/yproxy.yaml -ldebug > yproxy.log 2>&1 & + +i=0 +while (! [ -S /tmp/yproxy.sock ]) && [ $i -lt 20 ]; do sleep 1; i=$(($i+1)) ; done + +cd gpcontrib/yezzey +make installcheck || (echo Yproxy logs; cat ../../yproxy.log; cat /home/gpadmin/gpcontrib/yezzey/regression.diffs && exit 1) diff --git a/yezzey_test/wal-g-conf.yaml b/yezzey_test/wal-g-conf.yaml new file mode 100644 index 00000000000..8de5c4c2159 --- /dev/null +++ b/yezzey_test/wal-g-conf.yaml @@ -0,0 +1,12 @@ +AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" +AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" +AWS_ENDPOINT: "$AWS_ENDPOINT" +AWS_S3_FORCE_PATH_STYLE: true + +WALG_COMPRESSION_METHOD: "brotli" +WALG_DELTA_MAX_STEPS: 6 +WALG_UPLOAD_CONCURRENCY: 10 +WALG_DISK_RATE_LIMIT: 41943040 +WALG_NETWORK_RATE_LIMIT: 10485760 +WALG_S3_PREFIX: "$WALG_S3_PREFIX" +WALG_PGP_KEY_PATH: "/home/gpadmin/yezzey_test/priv.gpg" diff --git a/yezzey_test/yproxy.conf b/yezzey_test/yproxy.conf new file mode 100644 index 00000000000..855c3fce7d7 --- /dev/null +++ b/yezzey_test/yproxy.conf @@ -0,0 +1,25 @@ +socket_path: "/tmp/yproxy.sock" +interconnect_socket_path: "/tmp/ic.sock" +log_level: debug + +storage: + access_key_id: "$AWS_ACCESS_KEY_ID" + secret_access_key: "$AWS_SECRET_ACCESS_KEY" + storage_endpoint: "$AWS_ENDPOINT" + storage_prefix: "" + storage_bucket: "gpyezzey" + storage_region: "us-west-2" + storage_type: "s3" + tablespace_map: + "pg_default": "gpyezzey" + "tab1": "gpyezzey2" + "tab2": "gpyezzey3" + +backup_storage: + access_key_id: "$AWS_ACCESS_KEY_ID" + secret_access_key: "$AWS_SECRET_ACCESS_KEY" + storage_endpoint: "$AWS_ENDPOINT" + storage_prefix: "" + storage_bucket: "gpyezzey" + storage_region: "us-west-2" + storage_type: "s3" From aa5f3f636ab1e6da2b4a6607dd072dc266dae156 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Wed, 18 Feb 2026 16:41:03 +0300 Subject: [PATCH 006/167] Add group access to CBDB (#12) * Allow group access for init CBDB * Allow group access for segments CBDB --------- Co-authored-by: Leonid Borchuk --- gpMgmt/bin/gpinitsystem | 1 + gpMgmt/bin/lib/gpcreateseg.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/gpMgmt/bin/gpinitsystem b/gpMgmt/bin/gpinitsystem index fa85d42ae3f..f8d42e87b11 100755 --- a/gpMgmt/bin/gpinitsystem +++ b/gpMgmt/bin/gpinitsystem @@ -1272,6 +1272,7 @@ CREATE_QD_DB () { if [ x"$HEAP_CHECKSUM" == x"on" ]; then cmd="$cmd --data-checksums" fi + cmd="$cmd --allow-group-access" LOG_MSG "[INFO]:-Commencing local $cmd" $cmd >> $LOG_FILE 2>&1 RETVAL=$? diff --git a/gpMgmt/bin/lib/gpcreateseg.sh b/gpMgmt/bin/lib/gpcreateseg.sh index 5dd0f5b0006..89f5f405295 100755 --- a/gpMgmt/bin/lib/gpcreateseg.sh +++ b/gpMgmt/bin/lib/gpcreateseg.sh @@ -106,6 +106,7 @@ CREATE_QES_PRIMARY () { cmd="$cmd $LC_ALL_SETTINGS" cmd="$cmd --max_connections=$QE_MAX_CONNECT" cmd="$cmd --shared_buffers=$QE_SHARED_BUFFERS" + cmd="$cmd --allow-group-access" if [ x"$HEAP_CHECKSUM" == x"on" ]; then cmd="$cmd --data-checksums" fi From dbec496f32d7e3991cc9ae8821fbadb88c039684 Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 11 Feb 2026 00:43:38 +0500 Subject: [PATCH 007/167] Add yezzey as submodule (#23) --- .gitmodules | 3 +++ gpcontrib/yezzey | 1 + 2 files changed, 4 insertions(+) create mode 160000 gpcontrib/yezzey diff --git a/.gitmodules b/.gitmodules index a7b61644ee2..9ebe8907c58 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,3 +15,6 @@ path = dependency/yyjson url = https://github.com/ibireme/yyjson.git +[submodule "gpcontrib/yezzey"] + path = gpcontrib/yezzey + url = git@github.com:open-gpdb/yezzey.git diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey new file mode 160000 index 00000000000..ba7467211db --- /dev/null +++ b/gpcontrib/yezzey @@ -0,0 +1 @@ +Subproject commit ba7467211db94f0017e0028a6541fc201f02634e From cb19aed8238a8e868e437a01c8f220b78adcc9eb Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:11:42 +0300 Subject: [PATCH 008/167] UseAnonymousAddress (#24) Co-authored-by: Leonid Borchuk --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 9ebe8907c58..f900edb6807 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,4 +17,4 @@ [submodule "gpcontrib/yezzey"] path = gpcontrib/yezzey - url = git@github.com:open-gpdb/yezzey.git + url = https://github.com/open-gpdb/yezzey.git From 6c3bbd3760046c828a9619353d7abd6adc084f53 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Wed, 11 Mar 2026 10:21:49 +0300 Subject: [PATCH 009/167] Add yezzey build option (#26) * Add yezzey build option * Move yezey to commit 4c6b5b8 --------- Co-authored-by: Leonid Borchuk --- .github/workflows/yezzey-ci.yaml | 334 +++++++++++++++++++++++ .github/workflows/yezzey-test.yml | 21 -- configure | 54 ++++ configure.ac | 7 + docker/yezzey/.dockerignore | 2 - docker/yezzey/Dockerfile | 131 --------- docker/yezzey/docker-compose.yaml | 45 --- gpcontrib/Makefile | 10 + gpcontrib/yezzey | 2 +- src/Makefile.global.in | 3 + yezzey_test/.minio/certs/CAs/ca.cert.pem | 29 -- yezzey_test/.minio/certs/CAs/ca.cert.srl | 1 - yezzey_test/.minio/certs/CAs/ca.key | 51 ---- yezzey_test/.minio/certs/private.key | 51 ---- yezzey_test/.minio/certs/public.crt | 29 -- yezzey_test/.minio/certs/server.csr | 27 -- yezzey_test/generate_ssh_key.sh | 6 - yezzey_test/import_gpg_keys.sh | 5 - yezzey_test/install-wal-g.sh | 20 -- yezzey_test/install_yproxy.sh | 20 -- yezzey_test/priv.gpg | 105 ------- yezzey_test/pub.gpg | 52 ---- yezzey_test/run_tests.sh | 75 ----- yezzey_test/wal-g-conf.yaml | 12 - yezzey_test/yproxy.conf | 25 -- 25 files changed, 409 insertions(+), 708 deletions(-) create mode 100644 .github/workflows/yezzey-ci.yaml delete mode 100644 .github/workflows/yezzey-test.yml delete mode 100644 docker/yezzey/.dockerignore delete mode 100644 docker/yezzey/Dockerfile delete mode 100644 docker/yezzey/docker-compose.yaml delete mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.pem delete mode 100755 yezzey_test/.minio/certs/CAs/ca.cert.srl delete mode 100755 yezzey_test/.minio/certs/CAs/ca.key delete mode 100755 yezzey_test/.minio/certs/private.key delete mode 100755 yezzey_test/.minio/certs/public.crt delete mode 100755 yezzey_test/.minio/certs/server.csr delete mode 100755 yezzey_test/generate_ssh_key.sh delete mode 100755 yezzey_test/import_gpg_keys.sh delete mode 100755 yezzey_test/install-wal-g.sh delete mode 100755 yezzey_test/install_yproxy.sh delete mode 100644 yezzey_test/priv.gpg delete mode 100644 yezzey_test/pub.gpg delete mode 100755 yezzey_test/run_tests.sh delete mode 100644 yezzey_test/wal-g-conf.yaml delete mode 100644 yezzey_test/yproxy.conf diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml new file mode 100644 index 00000000000..c1c41497a64 --- /dev/null +++ b/.github/workflows/yezzey-ci.yaml @@ -0,0 +1,334 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Yezzey CI Workflow +# -------------------------------------------------------------------- +name: Yezzey CI Pipeline + +on: + push: + branches: [ main ] + pull_request: + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + CLOUDBERRY_HOME: "/usr/local/cloudberry-db" + CLOUDBERRY_VERSION: "main" + +jobs: + + ## Stage 1: Build artifacts and run tests for cloudberry + + test-cloudberry: + name: Build and Test Yezzey Cloudberry + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + services: + # Define the MinIO service container + minio: + image: lazybit/minio # Use a specific MinIO image tag + ports: + - 9000:9000 # Expose MinIO's API port (9000) + - 9001:9001 # Expose MinIO's console port (optional, for web UI) + env: + # MinIO root credentials (required for admin access) + MINIO_ROOT_USER: some_key + MINIO_ROOT_PASSWORD: some_key + # Healthcheck to ensure MinIO is ready before the job proceeds + options: >- + --name minio + --health-cmd "curl --fail http://localhost:9000/minio/health/live" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + volumes: + - ${{ github.workspace }}/data:/data + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + path: cloudberry + submodules: true + + - name: Checkout Yproxy source + uses: actions/checkout@v4 + with: + repository: open-gpdb/yproxy + ref: master + path: yproxy + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin "${SRC_DIR}/build-logs" + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Install MinIO Client (mc) + run: | + set -ex pipefail + # Download mc for Linux (amd64) + curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + sudo mv mc /usr/local/bin/mc # Make mc available system-wide + + - name: Configure MinIO service + run: | + set -ex pipefail + # Add the MinIO service as an "alias" in mc (name it "minio-ci") + mc alias set minio-ci http://minio:9000 some_key some_key + + # Verify the connection + mc admin info minio-ci + + # Create buckets + mc mb minio-ci/gpyezzey + mc mb minio-ci/gpyezzey2 + mc mb minio-ci/gpyezzey3 + + - name: Run Apache Cloudberry configure script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Run Apache Cloudberry build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Run Yezzey build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + if ! time su - gpadmin -c "cd ${SRC_DIR}/gpcontrib/yezzey && make && make install"; then + echo "::error::Build yezzey failed" + exit 1 + fi + + - name: Deploy yezzey config + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh"; then + echo "::error::Config yezzey failed" + exit 1 + fi + + - name: Install yproxy + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/yproxy + run: | + set -eo pipefail + + # Install latest Go compiler + sudo apt update + sudo apt install -y software-properties-common + sudo add-apt-repository ppa:longsleep/golang-backports + sudo apt update + sudo apt install -y golang-go + + # Install lib dependencies + + sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake + + # Fetch project and build + git config --global --add safe.directory ${SRC_DIR} + cd ${SRC_DIR} + make build + + mv devbin/yproxy /usr/bin/yproxy + + #Check the installation + yproxy --version + + - name: Create demo cluster with yezzey + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/create_demo_yezzey_cloudberry.sh"; then + echo "::error::Create cluster with yezzey failed" + exit 1 + fi + + - name: Run tests + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + set -x + + chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/launch_yproxy.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/launch_yproxy.sh && cd ${SRC_DIR}/gpcontrib/yezzey && source /usr/local/cloudberry-db/cloudberry-env.sh && source ../../gpAux/gpdemo/gpdemo-env.sh && IS_CLOUDBERRY=true make installcheck"; then + echo "::error::Test yezzey failed" + cat ${SRC_DIR}/gpcontrib/yezzey/regression.diffs + exit 1 + fi + + - name: Upload test logs + uses: actions/upload-artifact@v4 + with: + name: test-logs-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + build-logs/ + retention-days: 7 + + - name: Upload test results files + uses: actions/upload-artifact@v4 + with: + name: results-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + retention-days: 7 + + - name: Upload test regression logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-cloudberry-${{ needs.build.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + **/yproxy.log + cloudberry/gpAux/gpdemo/datadirs/standby/log/ + cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ + cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ + retention-days: 7 + + ## ====================================================================== + ## Job: report + ## ====================================================================== + + report: + name: Generate Apache Cloudberry Build Report + needs: [test-cloudberry] + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Generate Final Report + run: | + { + echo "# Yezzey Test Pipeline Report" + + echo "## Job Status" + echo "- Cloudberry Job: ${{ needs.test-cloudberry.result }}" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.test-cloudberry.result }}" == "success" ]]; then + echo "✅ Pipeline completed successfully" + else + echo "⚠️ Pipeline completed with failures" + + if [[ "${{ needs.test-cloudberry.result }}" != "success" ]]; then + echo "### Cloudberry Test Failure" + echo "Check build logs for details" + fi + + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Notify on failure + if: | + (needs.test-cloudberry.result != 'success') + run: | + echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" + echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "Cloudberry Result: ${{ needs.test-cloudberry.result }}" + + diff --git a/.github/workflows/yezzey-test.yml b/.github/workflows/yezzey-test.yml deleted file mode 100644 index 4922f6eca84..00000000000 --- a/.github/workflows/yezzey-test.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Yezzey testing - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - - build_and_run_yezzey: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Test Yezzey - run: docker compose -f docker/yezzey/docker-compose.yaml run --build --remove-orphans yezzey - - diff --git a/configure b/configure index febd1b30169..6c94bce46bd 100755 --- a/configure +++ b/configure @@ -722,6 +722,7 @@ with_apr_config with_libcurl with_rt with_zstd +with_yezzey with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -943,6 +944,9 @@ with_zlib with_lz4 with_libbz2 with_zstd +with_diskquota +with_gp_stats_collector +with_yezzey with_rt with_libcurl with_apr_config @@ -11250,6 +11254,56 @@ $as_echo "yes" >&6; } fi fi +# +# gp_stats_collector +# + + + +# Check whether --with-gp-stats-collector was given. +if test "${with_gp_stats_collector+set}" = set; then : + withval=$with_gp_stats_collector; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-gp-stats-collector option" "$LINENO" 5 + ;; + esac + +else + with_gp_stats_collector=no + +fi + +# +# yezzey +# + +# Check whether --with-yezzey was given. +if test "${with_yezzey+set}" = set; then : + withval=$with_yezzey; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-yezzey option" "$LINENO" 5 + ;; + esac + +else + with_yezzey=no + +fi + # # Realtime library # diff --git a/configure.ac b/configure.ac index 6f6ba21bbd3..89876e69d4f 100644 --- a/configure.ac +++ b/configure.ac @@ -1373,6 +1373,13 @@ if test "$with_zstd" = yes; then PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0]) fi +# +# yezzey +# +PGAC_ARG_BOOL(with, yezzey, no, + [build with Yezzey extension]) +AC_SUBST(with_yezzey) + # # Realtime library # diff --git a/docker/yezzey/.dockerignore b/docker/yezzey/.dockerignore deleted file mode 100644 index 4e92238e609..00000000000 --- a/docker/yezzey/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -Dockerfile -docker-compose.yaml \ No newline at end of file diff --git a/docker/yezzey/Dockerfile b/docker/yezzey/Dockerfile deleted file mode 100644 index 0b84f13d7bc..00000000000 --- a/docker/yezzey/Dockerfile +++ /dev/null @@ -1,131 +0,0 @@ -FROM ubuntu:focal - -ARG accessKeyId -ARG secretAccessKey -ARG bucketName -ARG s3endpoint -ARG yezzeyRef - -ENV YEZZEY_REF=${yezzeyRef:-v1.8_opengpdb} - -ENV AWS_ACCESS_KEY_ID=${accessKeyId} -ENV AWS_SECRET_ACCESS_KEY=${secretAccessKey} -ENV S3_BUCKET=${bucketName} -ENV WALG_S3_PREFIX=s3://${bucketName}/yezzey-test-files -ENV S3_ENDPOINT=${s3endpoint} - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -ENV DEBIAN_FRONTEND=noninteractive - -RUN useradd -rm -d /home/gpadmin -s /bin/bash -g root -G sudo -u 1001 gpadmin - -RUN ln -snf /usr/share/zoneinfo/Europe/London /etc/localtime && echo Europe/London > /etc/timezone \ -&& apt-get update -o Acquire::AllowInsecureRepositories=true && apt-get install -y --no-install-recommends --allow-unauthenticated \ - build-essential libssl-dev gnupg devscripts \ - openssl libssl-dev debhelper debootstrap \ - make equivs bison ca-certificates-java ca-certificates \ - cmake curl cgroup-tools flex gcc-8 g++-8 g++-8-multilib \ - git krb5-multidev libapr1-dev libbz2-dev libcurl4-gnutls-dev \ - libevent-dev libkrb5-dev libldap2-dev libperl-dev libreadline6-dev \ - libssl-dev libxml2-dev libyaml-dev libzstd-dev libaprutil1-dev \ - libpam0g-dev libpam0g libcgroup1 libyaml-0-2 libldap-2.4-2 libssl1.1 \ - ninja-build python-dev python-setuptools quilt unzip wget zlib1g-dev libuv1-dev \ - libgpgme-dev libgpgme11 sudo iproute2 less software-properties-common \ - openssh-client openssh-server - -COPY yezzey_test/install_yproxy.sh /home/gpadmin - -RUN ["/home/gpadmin/install_yproxy.sh"] - -RUN apt-get install -y locales \ -&& locale-gen "en_US.UTF-8" \ -&& update-locale LC_ALL="en_US.UTF-8" - -RUN echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers - -USER gpadmin -WORKDIR /home/gpadmin - -COPY yezzey_test/import_gpg_keys.sh /home/gpadmin/ -COPY yezzey_test/priv.gpg /home/gpadmin/yezzey_test/priv.gpg -COPY yezzey_test/pub.gpg /home/gpadmin/yezzey_test/pub.gpg - -RUN ["/home/gpadmin/import_gpg_keys.sh"] - -COPY yezzey_test/generate_ssh_key.sh /home/gpadmin/ - -RUN ["/home/gpadmin/generate_ssh_key.sh"] - - -RUN cd /tmp/ \ -&& git clone https://github.com/boundary/sigar.git \ -&& cd ./sigar/ \ -&& mkdir build && cd build && cmake .. && make \ -&& sudo make install - -COPY . /home/gpadmin - -RUN sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - bison \ - ccache \ - cmake \ - curl \ - flex \ - git-core \ - gcc \ - g++ \ - inetutils-ping \ - krb5-kdc \ - krb5-admin-server \ - libapr1-dev \ - libbz2-dev \ - libcurl4-gnutls-dev \ - libevent-dev \ - libkrb5-dev \ - libpam-dev \ - libperl-dev \ - libreadline-dev \ - libssl-dev \ - libxml2-dev \ - libyaml-dev \ - libzstd-dev \ - locales \ - net-tools \ - ninja-build \ - openssh-client \ - openssh-server \ - openssl \ - python3-dev \ - python3-pip \ - python3-psutil \ - python3-pygresql \ - python-yaml \ - zlib1g-dev \ - rsync \ -&& sudo apt install -y libhyperic-sigar-java libaprutil1-dev libuv1-dev - -RUN sudo mkdir /usr/local/gpdb \ -&& sudo chown gpadmin:root /usr/local/gpdb - -RUN sudo chown -R gpadmin:root /home/gpadmin \ -&& git status - -RUN git submodule update --init -RUN rm -fr gpcontrib/yezzey - -# Fetch latest yezzey version -RUN git clone https://github.com/open-gpdb/yezzey.git gpcontrib/yezzey && cd gpcontrib/yezzey && git fetch origin $YEZZEY_REF:test_branch && git checkout test_branch && cd /home/gpadmin -RUN sed -i '/^trusted/d' gpcontrib/yezzey/yezzey.control -RUN ./configure --with-perl --with-python --with-libxml --disable-orca --prefix=/usr/local/gpdb \ ---enable-depend --enable-cassert --enable-debug --without-mdblocales --without-zstd CFLAGS='-fno-omit-frame-pointer -Wno-implicit-fallthrough -O3 -pthread' -RUN make -j8 && make -j8 install && make -C gpcontrib/yezzey -j8 install - - -RUN echo ${s3endpoint} - -RUN sed -i "s/\$AWS_ACCESS_KEY_ID/${accessKeyId}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$AWS_SECRET_ACCESS_KEY/${secretAccessKey}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$AWS_ENDPOINT/${s3endpoint}/g" yezzey_test/yproxy.conf \ -&& sed -i "s/\$WALG_S3_PREFIX/${bucketName}\/yezzey-test-files/g" yezzey_test/yproxy.conf && cp yezzey_test/yproxy.conf /tmp/yproxy.yaml - -ENTRYPOINT ["./yezzey_test/run_tests.sh"] diff --git a/docker/yezzey/docker-compose.yaml b/docker/yezzey/docker-compose.yaml deleted file mode 100644 index 2562ebeaa0b..00000000000 --- a/docker/yezzey/docker-compose.yaml +++ /dev/null @@ -1,45 +0,0 @@ -services: - minio: - image: quay.io/minio/minio - command: server --console-address ":9001" /data - expose: - - "9000" - - "9001" - environment: - MINIO_ROOT_USER: some_key - MINIO_ROOT_PASSWORD: some_key - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 5 - hostname: minio - - setup-minio: - image: quay.io/minio/mc - depends_on: - minio: - condition: service_healthy - entrypoint: | - /bin/sh -c " - /usr/bin/mc alias set myminio http://minio:9000 some_key some_key - /usr/bin/mc mb myminio/gpyezzey - /usr/bin/mc mb myminio/gpyezzey2 - /usr/bin/mc mb myminio/gpyezzey3 - " - - yezzey: - image: yezzey - build: - context: ../.. - dockerfile: docker/yezzey/Dockerfile - args: - accessKeyId: some_key - secretAccessKey: some_key - bucketName: gpyezzey - s3endpoint: "http:\\/\\/minio:9000" - depends_on: - minio: - condition: service_healthy - setup-minio: - condition: service_completed_successfully diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 8d95a14f876..af9862530d6 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,6 +35,16 @@ else diskquota endif +ifeq "$(with_diskquota)" "yes" + recurse_targets += diskquota +endif + +ifeq "$(with_gp_stats_collector)" "yes" + recurse_targets += gp_stats_collector +endif +ifeq "$(with_yezzey)" "yes" + recurse_targets += yezzey +endif ifeq "$(with_zstd)" "yes" recurse_targets += zstd endif diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index ba7467211db..4c6b5b83735 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit ba7467211db94f0017e0028a6541fc201f02634e +Subproject commit 4c6b5b83735320dda01e042631a851336300a3ca diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 77b58e7aa76..457a4a0944e 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -271,6 +271,9 @@ with_zstd = @with_zstd@ ZSTD_CFLAGS = @ZSTD_CFLAGS@ ZSTD_LIBS = @ZSTD_LIBS@ EVENT_LIBS = @EVENT_LIBS@ +with_diskquota = @with_diskquota@ +with_gp_stats_collector = @with_gp_stats_collector@ +with_yezzey = @with_yezzey@ ########################################################################## # diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.pem b/yezzey_test/.minio/certs/CAs/ca.cert.pem deleted file mode 100755 index e9f2f1d73ee..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.cert.pem +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIE/TCCAuWgAwIBAgIUU9e6chP84r3iZk3JtvnWb1V2N1YwDQYJKoZIhvcNAQEL -BQAwDTELMAkGA1UEBhMCUlUwIBcNMjMwMzEwMDgzNTUzWhgPMzAyMjA3MTEwODM1 -NTNaMA0xCzAJBgNVBAYTAlJVMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC -AgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wDvzjMPE7KmONU -8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwfFL2F5mKHgeHQ -n3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb0JOZis82d+lh -kcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6KNTm42mubVlX -kBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW7aloU5zPBhYf -BaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXvEUPwK7BZzgZy -MerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujDv9l6/T3sCmRv -tWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqubkdJr1Ro8yI5J -m0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRDajZaZ55DopmF -+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh9UvDhjKS43bQ -zSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEAAaNTMFEwHQYD -VR0OBBYEFJGDr6xmoKJFU6cgS90aFg6lUGbhMB8GA1UdIwQYMBaAFJGDr6xmoKJF -U6cgS90aFg6lUGbhMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -ACj87ymjBlgY9UZTUbudHREPPXfqMi2TgWt5hygQSiTrNeQOodnq+Swp86qX/y8w -xtnvc+iILfFnh9ZevHKmLx+JziN4kD4ywEpHW7zS7c3+2QEjIZUwj5qlIg0ByOBd -0M/kpimmuTwlDylBaY12GcFlZcsbuezzm4hU+0qoCV/zi2DvSdAPKXMAeZ3lOkde -PUYJUpRz/QkkxEhSdM3BQYI51mUiltCHMhe6COoN4MHV7tix0Pj9vPjhAVN/4sot -2PgUiCwY8eNQugZhpTosMTSBLZvg/EKG+4slY75/voNTIxWHAHmnPMOAzVgNTya0 -/eP6NB3MCjFuY2E+fGox9YTomjI5oxBr+1LlwVy7wbwXTrgBz9Z4izScAsVbPrk6 -jSrqNeNWK1f+JVnYZkjgPGgPaQVCJ22vdLmkW7U/ATdeedQS3RCApMnb9VCRTUaO -eY4ccuEvj0huhdcUguw6fBjrhPjoPxKMn6S93ginW8Wz9vo8qLkEg2NtQDFu1Omb -cJM5F8uLRr8NotPV5QPg1koHeBv/N2WTRZiUoavAogR9XdyOtrB8+MBu1nsp4Goi -7/suv9XzMJ7IpgXiQfCM++1x7oooyWWdeFTCzqNDJ1IbQDeOCc9cQgeOAPWcIqWO -nAWt08+eToI1YUvjl6UT0bpVaJEACv+/HfBr1T26u4Jh ------END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/CAs/ca.cert.srl b/yezzey_test/.minio/certs/CAs/ca.cert.srl deleted file mode 100755 index 977dab4a3e7..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.cert.srl +++ /dev/null @@ -1 +0,0 @@ -53DAAE29A25C5BE967A9E9631F0572E17AC92211 diff --git a/yezzey_test/.minio/certs/CAs/ca.key b/yezzey_test/.minio/certs/CAs/ca.key deleted file mode 100755 index a0fc44ee422..00000000000 --- a/yezzey_test/.minio/certs/CAs/ca.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAwJuy394cK127yT8nGHVPKF6TG6xL0WpxahyaKwIYp5lbv9wD -vzjMPE7KmONU8GhCFUdEJTRqBkaRdZNYxnOUxufU3+jIf1hq1Csg8q1NXICVWVwf -FL2F5mKHgeHQn3FaJM2pZQ5iIWFY1c18MgV8qqNWbtyLeppcyZOL9duLM9A8XpYb -0JOZis82d+lhkcxzE1XM+MZEgZfHImh0zod9OMtSAOwQzVXpiA3JO/eHkLQGYcy6 -KNTm42mubVlXkBcu/BplnP7gXGOYDt/JyRhGSLAfn762+jRbAlAvbPzOy67hc4pW -7aloU5zPBhYfBaTxM9UPqPtyp7Lxkp9HL68QXtm5MobDuDtZ6ePQtHgHrl7P7PXv -EUPwK7BZzgZyMerVhxIssutA2yBCuu5T7dMSwIsUdvXtgdHRdHDwn1D/V1CxnujD -v9l6/T3sCmRvtWPwTOCUf5BLLw6N6TnSsVR5I9NALKCLYE8LsfCuLdyi363JZqub -kdJr1Ro8yI5Jm0GX5pypwZJPV2Ivt6kKVTQiN2hoWNe+3TNPS+7ysqit37s71YRD -ajZaZ55DopmF+oIYdA3MqUZEVZyKFifWvo/l2gYarlEtcEJl++OwydirWLAjCPHh -9UvDhjKS43bQzSlRC+d4CfRqXftmETHVAxMokai3WvAdUpJrW2RrjiuR0MkCAwEA -AQKCAgAgemC4RTDE00J2FfMWublGWmQ991i1kFhdh0Mr22ei40ZIXOY42W/+/15E -V5kcDMiP4/uGtobmVgHzLIx8skK1I6SOuScN6i/hZQBiS3zPC1OjxNfs3GR2y8iD -yzstl6SWriNRShKcBFlBfCvkF27FK1PIz+GpI9xflUS1iXa4nvV/EZrRGgJ7GKPb -pnvwZORGr2In1O76V0iZ8bk4ljo0WHyUcToIFeOSMJjtRrkSWnj1BtuhRP1F/a0O -/VC5mF8w3Zai2YulqJmccHoLMc+wNBqxCiy6lhd+lVzZ6OtKB0w2+m3cF4PjDX8P -TK2gewa9McE5QmU8B/2aNsd/L+r3eGEvWAF/1vRq6NcrFwigq8uCTtgw9edRlDnm -RvICkfAbrwhNaixWwqBVQHoy53H29TohxGNNKa6TTKeJvYEdYKgHx55TxkB9X9jc -iSisqb3fgEl4Yh1Izpu+6nULOqdlldfkKPgKJqVB1AT/avR8J09zmMvW5fPa6fFx -alZ1iVahR5bIFEu1lXygsrBP6N+K/ogyztg7ZKLTIN/FguwMKnXMaUbN/Y/ZZXV1 -oGil9vHKnDrRnUGfcm9tyH2Ddcy6RDoDz+O4cYgMGxDhHran2cicVY1q+Yi08q5h -Napk1phNra5HIHnNHwMxQ75ZKZZ3TOGJL+HMF4yRDj19C/6sAQKCAQEA8a9ZQhWw -0vhZENmSYZgGZLa7RZLSbBzQOX/cetdI6/kvmZVcMvNz4q0/UI9XLkqokL1wJiku -O0zXkaVrBVAsgozp4I3oFqwtcAAGw0KwF4FDAS36k4gkE4SmIUl2eI0XMZCPQIKp -3TB81+XdBITtwfPl5yG+IZDkXNu16qUHEhnhvs/kKhMr8flhFC1J4gdrrQhfuRHY -Jv8e1RLJzMhu/ErRjh82LkzB6m3jp0YxBeIA+9Kkw+OX6SlzRbJPirKxJTaZnB8o -wQmzOy1kTRG4qjKswjdTbzf6549721i8QHwSpwPI3NZQhlSkfsvZ5QL4qPW0nRta -m76YeLlS12yQSQKCAQEAzAQz6OcE6yS2q5UfTZluGaU54Zkm0YSnS394pitJpHoh -JSZlvkL1DzpacquDxa3uQLDikai5TqpNnkuufeMJf7I2ygg4n/v4OFaE+/qj5uNA -3QnL3BVT9DCJ0JvQ1qA5Q/6P5WpUHYB7JHBM9BpaE8e4xocJyWSdcSJDaEXns4Hx -WzhpBdVpPSamqB0VHYg1bv6OGFPfwUaRafWhNzljtxbY8RYcz7IfPmnLImFePTtZ -AjzIoAwUIRFzvmoduda0kQKogRVoEeaW1q6ebPUjYjIZvohnpe27EvgCiTNkcaSf -C96uIxHrSvI8114z9CBXer60xQ0Kz+ds18LtY6w8gQKCAQEAkP/JxlsrHje/f9t4 -9jJ2S4BSNLiUpCZZStYKWmzFJEX5J+SzTyI+uZWFcfi9rlk+brApE8wLH6rHfmtH -HQXv3ldajc21m7yq+hIZ/JYK/d8gaxnBxzebpVYlMb1YZZUIgEUhnOuHq9vGWuVe -x7JUztNccGIPJyY9y/RJXUCrUFHU3Vzun8umxuL+OlO9iu02zbZDb85j52mSfvVp -uwHZjGX6+ZCCOh71DIfnWFlFWikwu+Sx05C9eDbVINCM5kK1AwWR/Ve4ZLBEJtHh -5lcmen4ypcb5uLVWRA0SmxPOxcVqj2c24D94Sk+H7UayMLKqqvvW45cgsmYUJgHR -0MsieQKCAQB9goBk4erWtmliuYTeemuPf2RSc6O79b3t5mfU4oCVnUTS1AJ3wD1+ -tsl6DiYs8MnIJoncTk5iJMdHgQvCCnCHjJ3EQLaFRb/4+NErK5C1tEztLt+pb72M -VmgSXCloQH26ZNslqfpBhA895ZCSA7wyuwXjrKPKsAlj1k5d0dOvTVusYNHLcvUh -V6vjdLDO0EL/G79THBZlkwJWi3Q4wyejNX0VJCNpaw1pmjAL4JbXWLFzfO13+LZR -eakZFbNf5sSDCX2cnAzAJnnZbOet5El2WZgY7VXGcLBMBSOaQHGksD/gT4gVrypv -mwLvA9c2cscejkArkdB7AsalHhho30cBAoIBAFJBO0RU7o0S+F6KHIP5aFbItcUd -NfUgoJTAFUD3EnBirvDv0pu8T8zkgKf7PRFkZQIOXocvpX0Zy6N7fiPbvzTA/vH3 -mFqias89pTUAgv43R8ZsAC/qlozUuByegigEz2zeVd34w7MdkgGo1jnqmijAIXZE -INBo0swkxAbix+W1Pur/yvGUpC6xu3ISmdrn0p20B7QhyuoqC3ea/az7ePwx+Pu9 -Jl8tzMujbHNHhw+OQAQOPHi6EUPs/H37euj3G7oBaVUwXJq3Tbwg95W5Jih+CgTB -Sbe6eYpR/j/SYGwbS6/DbHi3IjvblN+2pSPI05JvXMhLC/lAeqcdVJAgTvw= ------END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/private.key b/yezzey_test/.minio/certs/private.key deleted file mode 100755 index 9d12f84dfa7..00000000000 --- a/yezzey_test/.minio/certs/private.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/ -mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm0 -7Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa66b88Dz/mHDK -JTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6ANfTXmhVuVm4X -7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOzGoAY1iJbHuXV -tzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQdGbdmKiwu/NL -oc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxG -ScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U51xDRk6qkQbO -Ah5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1co+0Svm9EEG9f -bZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+ -flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69fMGozzkCAwEA -AQKCAgEAsWK7PvzUcBzosK6GW6/HloJCLniOpyOS50LTisfEnZ4qGn9lElrwv1X7 -bliaXsutz+rFbHdVQVE6fhU723DtlAhaUS2WC5n83j5aP0Lv93qaQIkSLj+DoQuk -UGIWetQQoPFG1gEjXoAV1k9tsFiXTGz8RpnDmNb2PMW1u1AF1G/gygh5Ape0fs8C -YwvMCnfL/eEqGRY8D8526+09YGv9ijXle32MLLHDuHWdfz0aPayzHIZIvXf2Unrr -vUwJAZ/ONz+Obj0etVgDpDrDD5SCWVer/Jlj/xT2Dfg0W0NBYkENHpJRJwyQNYJu -xWe7SIKLXslY+JWavhhf3nRkjOJWIGtgix6Sbb6K3TFNu65t5nOVr9dyZFcW4sSD -JpyUjDK6mbpUKgh3yaU9QXQSY0bD48mTt1UB2vmh/dA9OIRRUXr+V4MKPQBtJt0V -7ay2P95oWSHM+zpzPw2Bv6R6s78kaFisD/IqVwt95NMSiVTb3rNsJLs+KNKjuoV0 -EPeZuiQvDZDNi2+pKC+3tT65+sECno6ZSZdvn7naRYy3X+QWbOY/zG9mXIMyRXSs -oDXPH17838swDO7YrmWDvHO2Coz72eEftsovDql+D+4w6lq5DZygbHiSO3yyB4cR -AbI1hQ0nneQC8Gi+YPNEguAz5Sdvys91urZ1awllsTpG6gFPuAECggEBAPROYIyi -4mjr3MAUU0h6FHQ8uCARknM2NOZGDpyHmOtFjdTW47xGB3E8Dhyci0iOYjgwhcNe -lx/gZySKuCbC4HbxWUzLFYOOYRpj41oIZW3kvnkc7vVEst726CO+cVtTc8cjYXQL -pFQO3wN9C+S7OXIZOSO/P83jHm3bG9vQ1Kgh7gMBgZkwx3NMt2d1slEo4NViXZ5n -1960wWer5J6lQtpgwJYik7tZZBXkGA8QrhvzqfAGLYUQc3chWSHqlRt/+YJzFXZc -JYhpmCOs3jefC1I0T4wsxTJAhv3xnlGY5FDFIsHXqkCgwsqU9rYNfNQLLhuuwG3t -kR/sZp1eKmkDCjECggEBAN+Z1DqZIgd9rXTeUICbzlQy4891BhCWy3F20CTbzdfv -7EN49uhaD/OltJ8LJcDqm2F38Xjz4+svGfsYkqe0EMV1IOkdhnf29uGcWb/kWGGX -FtNRL3QhKntouVqqsJdeFNcvbPF4RZPUOQTiHjH2U+nTA/KIKU/nxSqJwR3fj9nc -v05k2jv+eoodDx4Fs/zS/cYzA0bjEXZlO6fS+MSWJwQGVbd3lqieo0FVuD3Y2RVs -nQidKUOm/qTE1/r//ggr0nX/GD6n2gRyUTV5yHIoZ/ENCOuxu38qyOk5ko8knPeo -IGqluaaTCyFav72vS9IbWVUKicdmzaLaYVLG1EzMS4kCggEAbHhoMckYUZF3f+kG -WUWq0zkqX0KuDW1h62PrlOA3qy5EnN2UW8GUCFirw1RWGy7suRoCKg5TdxnBcd4N -iVg5JVZfWdNJiBGtV3RGO3FC55oKX+fSyR9pc8mYpFYoKm5RF3fECywoGBJKPlhE -8ZeXF+vPDOobQCS5G/mO0vhTkxsNfFQs4IFh7PWA8PS26YYG3XdBLdXZaM5EmO0f -Irkph6AxofdUNHiqEXYmoHm9dDepmiKED3KQcbJiysZG0eDuPVcvjk+3cmu/2wcj -2vpIooULVKeEHp270gB3VK9Xn+0HU52Xv/4gvqWJLKesZtFX9X6Pfb9fEhd6MCDh -H6Lz4QKCAQBVTk/cp02HRBhoDOTzm/2ku+GT5VaR/6XQPP7+AOnQZ1VhDIZOcQXC -88YsI0mdBySOk/8JISskL80esepJlvYLDzumFECYBh18R3UqM4jQep8XsdKD4J9f -g72rbJGAvkD/M7XBjhwlYQL77PSOJScwfvPzlKUGOitLplCKAB/Wg1RtKSblWpoP -lTSORKi9XRW6hv3KDpk77TVMOr3z6kEqVOxg2XweCp/ILlbjKZbwiIaEocj7aXuf -CwZHWWDmZymdINx0Ev+nUKHQ4AxsGSRGn612llEtG/qYmAFlIRfNmFP9vApPFaxa -Zk/eo0EvAIYdXq7f50Wuytf0h6y5O/vZAoIBADrF4jHV+kbwWso1Rj4SA9xjmDSS -8v99Cr8YReoEwJoRo7sLa4XssY+deIvvR8OUwHmbPWjzDXeJFSMvJd9H7hytPvTW -wEPXd4eTRwcVo4d2ZUxszmJyLp5Mx/0qMtI2XkYPRfulPhXReYjkezjJ1mIzQiwL -RS4vwafBs1LvmbAM9stJt1K/XoI0e4kTYd9KyruKRW+JeuCrxcSV+O4nMRyOiCIQ -cHFXSlSHDyQo4z46rp2+IhIt9/Vq5DQ8hhbAtIjqa/ndFUj8FaQs6sOQq4410+LK -OCT7UGhKdY9I6fV/O4MAiLzyXcXNo6SRZ0YmPfam2v3gCqdDHLneLsbMoJ8= ------END RSA PRIVATE KEY----- diff --git a/yezzey_test/.minio/certs/public.crt b/yezzey_test/.minio/certs/public.crt deleted file mode 100755 index 4da1db77d25..00000000000 --- a/yezzey_test/.minio/certs/public.crt +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIE6jCCAtICFFParimiXFvpZ6npYx8FcuF6ySIRMA0GCSqGSIb3DQEBCwUAMA0x -CzAJBgNVBAYTAlJVMB4XDTIzMDgxMTA5MzA1MFoXDTI0MDgxMDA5MzA1MFowVjEL -MAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3QxDTALBgNV -BAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMMAnMzMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR+rDY2+IK57DktPou0KlJ -+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZHDTeaYAmKYe0Wcqey+bzA -KG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz7pZArnIrZdOwixBiu8oa -66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKdMqR99irHKvGtC6TN5o6A -NfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhyaoEBw/aQmltfuEilMxrOz -GoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zEYqxVurYQTz0cWQZxo8BQ -dGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCIJiQLzGsUW8Jq6kstiaE7 -IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh9CHfNIytESa1zb1H/d+U -51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ecmaGmnAA3SYRZYV6yYf1c -o+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2LKJ5TVoUCV7S+Cz5tXFyJ -yHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERUFAlEAvag0+06qhbVyt69 -fMGozzkCAwEAATANBgkqhkiG9w0BAQsFAAOCAgEAipaTRZxvDImsu/pwxSHEkKFz -ndKt84oEeuWRBo+sT5wZAcUJicz0tHVULwhXiqwMRC06LBqFxGzI/srokK2QiN5B -eikQvWEsr08tZuVm1ewNit6JzlwbaLOJc4DSTgJtWXKjWaIzqWsFl36ViLO55yTb -NgMgmpDhBl6hQ7yKtaSVJ+xiMKSd9nz6gRmJMz7sLchFmy3fTYtayUJcaDsFjEm6 -yTYM1oWj05xVcZJtASB8Bcp+XgeOdNbwvvTGrM2ctDC1GkPCK4X63GJfAuCv6lnV -ggzr2Z8dg/YVLCbmyvJq4n+rwNbVDebQBzRzyU9id0dRccyfPQzB69LPYcrMFzd7 -h97i5Vk7Ar66VeKxdw3Lbyl91yTUy7e7EdYkBrlPzMohRcuCEsFPaUdOoiy3dmUX -PeaubgtJpaNlVbp08rdGAgcrY4aqmvVM+tlKWfgiEnVmQ4vKTcrITnhintvttXyh -GrddsvKAE0m1rDpfd9BqXH4FEydFHoL7oMpKHnu9LUQsPHnwGvpq75KUN4j2nP4P -NIEmmqOr5SY9zcp9HQApxWPscQckb9aCIDZ8MMrceNSxRtU+bw0xNs+IILGx6dNA -cK+bDLPtMneno6XXDqqiitE2ohXf5WIeB3kPLOsp3awVhr/g+pqjMSSjyUEC+KXr -dkqHW3oKT52eCNzR350= ------END CERTIFICATE----- diff --git a/yezzey_test/.minio/certs/server.csr b/yezzey_test/.minio/certs/server.csr deleted file mode 100755 index c223c34efb9..00000000000 --- a/yezzey_test/.minio/certs/server.csr +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxDTALBgNVBAgMBFRlc3QxDTALBgNV -BAcMBFRlc3QxDTALBgNVBAoMBFRlc3QxDTALBgNVBAsMBFRlc3QxCzAJBgNVBAMM -AnMzMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1WMTewnKOrmE6ceR -+rDY2+IK57DktPou0KlJ+Cir5RjsCrh/mD8rNCJCVQ0ytV8EHUlnfJBcSnZRuKZH -DTeaYAmKYe0Wcqey+bzAKG9+W3kNvFm07Q/MIEAU4eGoeMtZpG91ZU9Jml45siJz -7pZArnIrZdOwixBiu8oa66b88Dz/mHDKJTgex9q27mYFERWu51ORAIchCLIQ9xKd -MqR99irHKvGtC6TN5o6ANfTXmhVuVm4X7Q6aeeDdmsOoP/KKmK/0yYvTunyfEhya -oEBw/aQmltfuEilMxrOzGoAY1iJbHuXVtzYqkEomrmGXa7pStUoq3Ruu5PLVU+zE -YqxVurYQTz0cWQZxo8BQdGbdmKiwu/NLoc6WA2dUHdVKB9cDnRQXfKXWO8TZOKCI -JiQLzGsUW8Jq6kstiaE7IQL9LBZbSWxGScgkneDE+CucgSjD5PwDMZ5R6RXPVxdh -9CHfNIytESa1zb1H/d+U51xDRk6qkQbOAh5gt241mlekv1HYarRkwx8v0iiRa6ec -maGmnAA3SYRZYV6yYf1co+0Svm9EEG9fbZMHLmTExpz6xPwg5EsJ2jMgsJP6My2L -KJ5TVoUCV7S+Cz5tXFyJyHmtedSTaeC+flt5A+9nYQKuk92mh2t/XaSCRDFugERU -FAlEAvag0+06qhbVyt69fMGozzkCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCw -gv4AQ9xh+LP16NQ0QCZ7QM8y6cA7jb7yqc8G+gvTjbacsuxpfI7qiJHx/Aon/ReL -D3Lra+HKzQticJaqFQEiXE6kXWB4DndP7hY7MF0R3xlBvNWyS8Q6Nr3m6IQiJx1Q -qd2Fgc7341wOMf3XMW/E1XUEsnzL8pWrKI+lvHAMI3u2KvsqwTtWtFJ4HMQoab7D -Xi4QYVk/DMF/lirPgcm1xnDZP07f7lzIGasO42XOBW1nV33w3bqVjRfKLZnbCs0Q -IOecmfseUBER9ycTUnFHN99BGx1SmTtXmqoeQIwijIvMoBNtEsOskqhW1+snFlNO -st9pyjwhuASvSWDFbEsaC1tl+5oTK31XMrxs99TDRoPMY1UwVFbrXrc+XWP1MSwg -0SB2c2DnLwGT9Lp9w1+epn9oa95B5JskwRp86Lbf5y+XV7W3Vp/vCzQ+db2sUHuT -bEUpNJl/KcxxseMMkpIJy83w17RhlRsgMKvAQtftYWX/z/RHyGKWHHPPoxEJaKyc -hm+X65QJd8QoOd/IXnkCMTQZocpQxoKBbYqdTBDxYfXnTILWHRUb0OfmjS9fEmBE -UqYdwNawOIXKlPsI/JseO132C3TtxjmkqMSNixTFyRvpoqy+/wFul3QA+yF2eCM0 -pfKcr8QsHn8WEVqjEmyQiM1ixrDIgIDc1jurvLX2cA== ------END CERTIFICATE REQUEST----- diff --git a/yezzey_test/generate_ssh_key.sh b/yezzey_test/generate_ssh_key.sh deleted file mode 100755 index 1df4cf14b60..00000000000 --- a/yezzey_test/generate_ssh_key.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex - -ssh-keygen -f ~/.ssh/id_rsa -N '' -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys diff --git a/yezzey_test/import_gpg_keys.sh b/yezzey_test/import_gpg_keys.sh deleted file mode 100755 index fc351a3b416..00000000000 --- a/yezzey_test/import_gpg_keys.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -ex - -gpg --import yezzey_test/pub.gpg -gpg --import yezzey_test/priv.gpg diff --git a/yezzey_test/install-wal-g.sh b/yezzey_test/install-wal-g.sh deleted file mode 100755 index bfad4e82a58..00000000000 --- a/yezzey_test/install-wal-g.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -ex - -# Install latest Go compiler -sudo add-apt-repository ppa:longsleep/golang-backports -sudo apt update -sudo apt install -y golang-go - -# Install lib dependencies -sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake - -# Fetch project and build -git clone https://github.com/wal-g/wal-g.git -cd wal-g -make deps -make gp_build -mv main/gp/wal-g /usr/bin/wal-g - -#Check the installation -wal-g --version diff --git a/yezzey_test/install_yproxy.sh b/yezzey_test/install_yproxy.sh deleted file mode 100755 index 9e3abd1d9cd..00000000000 --- a/yezzey_test/install_yproxy.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -ex - -# Install latest Go compiler -sudo add-apt-repository ppa:longsleep/golang-backports -sudo apt update -sudo apt install -y golang-go - -# Install lib dependencies -sudo apt install -y libbrotli-dev liblzo2-dev libsodium-dev curl cmake - -# Fetch project and build -git clone https://github.com/open-gpdb/yproxy.git -cd yproxy -make build - -mv devbin/yproxy /usr/bin/yproxy - -#Check the installation -yproxy --version diff --git a/yezzey_test/priv.gpg b/yezzey_test/priv.gpg deleted file mode 100644 index 1b9727a319c..00000000000 --- a/yezzey_test/priv.gpg +++ /dev/null @@ -1,105 +0,0 @@ ------BEGIN PGP PRIVATE KEY BLOCK----- - -lQcYBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb -aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA -0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH -vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP -JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q -ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa -OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt -/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d -w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 -+TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ -HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB -AA/8D1gsTYmHqNYbLNEr9nsCIt6XL2Abg6s51Vj24z/cd6HJOhQEoeDxdWNav6NL -SPh5KDJNsCk4JvnqAlKgdIx6CXLe9XQvCeB9hk3Zae/91GeYFb9sN8GFkppf7ZC2 -TEf10upfjo6USnc9lkEwv/R2Hjyerk2WX+rYldvol2JneF1hLV2OuLyyY4Hm9mQx -qR30f2/u4gXBzFSn19A+hYn999qDsBiUnk4/mZ590tZwjh2mpixxTzS9HhbMLCAL -8w01rhVVqgZTyhhK65aP1cuZCgiPsy/QSDYzkz3F7sRcmBxDyPul85koWbLPk99M -C4bY7xMBKH9m8rRcXdCGiAuBKRlIZFQcyI7Lw4WxqxdJROAjhYb+FXDKSst+cyw8 -c0qP/UbqnNaZmWQ0w95UMlwu8MZ6sJK+CzaLkjt4itkN27/DCfxyfuETqEiTO1ba -yJoVDE5MH0CQLIlByvJAU9AmNd1vLL9CTzm39m8B+eOrPR5n2khu85D+VR3vWZo7 -XnM5RTXhKTMfGFCSunNVlnABEy1eFZYFOogZOkVYdqxRTiZccDIZgV+spN/op4kA -SofCehr1xxdd3DXRar/GOz8J2jrWyqaQzhsD4kKl5nBM3NH7fAPsBBv0VH1DE0OM -ZsiB0yQFyHTWeQOcNla37pD1H1DZa4fpsL6m3BqSK38cKcEIAOUr9faEnjYPTAoW -TKWpoqt2ghyGsklrdYVcgb7pINam1UV0f4uvMBv702ZnLwoC0Rn5ITgPXH0a0MDM -H9yfWZgb6L/mmE+B81D+EDUkb88qafXXq9TMspbwbSuY/DWFTZPVfRDWYacr/IPq -XdtZqNBR2BWDeckgBcnEwg/PBn2HrrpfWab/D9Lt1QFY5baCuFevib7/D1DxO9qq -57iA0IDT+YfydYawwixvarkCNmUfkDowhR7DqHWDppWt5n7rxiTuE7I7C88pV4mV -rVTvXeMh6ooB3m2yW1FhsTgT0DrWxul1glCASQ+TfOks6dcDMvcFZFSzWT1r2rR+ -LqNNpDkIAOa9LcZEaRV86QLLcBYpPnaDAFlZYU/1b6QRbU3L77rULvtHSVegEApp -7a1mDrx1bTp5Z6TGq5ls9bMHF4tg5IcBLDHsO6dYAl2vAbYuCwjfPa3hu/oK+J5F -9IQLrvS2vM2shqhIfIbKglXwAWySLQoR/CsAMgH4vk/R/08AJQRyXndaY3rLImQg -ofUQaRxl4950wO/RJSQ9RD5q1bvseJ6bZNlYu1SoEJ/uOGqlMbMP9DAUb7LnTwlt -+9Mdv/ibYBABnUh5hLLCc5lBgeN+Pd3AAEygvbK6rW9a2CV4vO5U0aBU7Um8WAUz -suVEQM/rMTJanxHAeHgRYMDccadTAZEH/2i/xd20U2Taorj+/Lrq21hQJ1jKpyN+ -CnglPV83pGX1V1j007C2zhO/YB5qTh9ztZvtwkuX5EoUN9mGRZ0t1K3xbmwlDQbB -D1hofRcN0U2h+ZXQ+aID+gswLhw6jWatHiDmhIGSPzopR/1oZrQwS0CJuqb3XQFP -lAgeMPEuFufIf7tijFLL3PDuTPnG5lndI6+uCw+BhkfgN7jAHHREQlolGgzo3mwW -htGr6cOd9W7a9tGN7nHb/yVH22gnXs++wyC8OoeuEgAqQL2CyNOEo3EyN1K79QH0 -7qaKg+XHJy/NymQdcALxp3zUNh3UuOd0CZxPudTVfD5j3cD5fST9Owx+CrQsUm9v -dCBTdXBlcnVzZXIgPHJvb3RAaGFuZGJvb2sud2VzdGFyZXRlLmNvbT6JAk4EEwEK -ADgWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbAwULCQgHAgYVCgkICwIE -FgIDAQIeAQIXgAAKCRD/Gb3KATgePXGWEACRiwzgx5EVSpQEKe79sRAkO9hmAB33 -D4dN4y+D+2kUY/paPg3etCdPIJQs6qFFu+HIWnCTGiUepcsBeUvRpTDwguP6MtXx -TObmaU20ZoSbZZjqnGZ5V9VlwCQjJWj3RN+7BYVsECPJGfEMW+M2nHX3MrGpohfo -KpacoL2kjVswvWocGHNIMHpVHlyGGUtwVHz3kO5HqwcZvWwZj189dKTMUSsfbXdz -723BDp+m/NLUi8Ki/BZpsRwSXqGbpdQ8LYaSJvtS0DY/rIILIOyIfhnHK74rgCip -0vfZsfypc58YrPrj9+pfF7Rx00m0RfGll+lD1Pha4GStS1PvtpdtRTvf6qsmArON -q1N7ha8YJHtPueTKln/7ik3jqir7vO7SLL+CTa/XfS1jaJhonHORfW+fekndXK34 -zrQ75cFn7D5BqqcTXu2hkTDbPqd9aBorWFU/vbqZRVH1a/RRcVU9SODnIwdCm06x -XLoujcr1daZd9GWT0PMvFHaehQ/4zbdd/pYJ9efU1zlfpdwxUpBWyfUEOGbwpHGv -7FjXIHSkHkTNgaOK5AP0FsSGfjAzTxq/2yLRQKm5f3GqWNOZ7nprLg85Tyxh/gdZ -rk8o3/pW7P/XDLU6jfhWV5YR2mxddaFs6P7D+jvRf3EVHbJHtcK30usIWjGDWyNk -310w6Q3CumfEo50HGARk248xARAA6LnhBgnfn0B405FGDbb+MOj/0dmmb81XwL52 -ZqbAIsomMUfrzzFHJb0PtexSsIr7I+NweVCSvc3UaGcMLF8pqOVnP4pkPXi+hBlG -eNeE4KQvMpn56nLsZ90GtpKYLLsMQB6VMm98+v4wCJ6mPkM89SPQlc7q9QFfC31g -trbb5Y//eDqCAATiVIdZ6nEFNKgNDMKC0UmRY+D3RaeElPpWO5lWDDldSjOZ9/Or -dg/TSdRviK7+NzdyOM8xUc6lQunG26qRH/bqQzHluBYMp4W3l71Ojf8YlSATkziB -dAdwpQ1MVx4rCk0o5nYuGf58sgxTIr6qIR4777Z7lanc9utcLeSbltkid+zATW8x -PFPG1ID6pNZ0JX2ttQ7HQAb3RfNDkZxGijqEnJyjY0nJON0/8Jcz46GkeUzkae0r -I/TOS7OASs13NWMQV8TsqDy9VMT+pcTDjy1Rad8krXtraK93WlIPWhb2vCSJkCMr -412bl2SOrtYUdu093yLYkEkrq6vZsl4zkIPrCbuEdJZgSn5d23nHd2ARsaj6D4KZ -RP9dg7A7RpEezNLAh0FDwMLe/nJpNfQdd9MXITALc0CwJ/x0sFBrj1+q7jELYtSp -zh993C4i9ySOaWKfOGV9UjeSLBYfExuxisMQyAIoD8hhNOHnqWVgcYqJ7R3w0ymU -96M29M8AEQEAAQAP/1WZpSiM8ilH1AlxmlRKFjYURaBAz6S44UmeZLt+IxbIxwKC -Yzxq8jHx1/kAyytve09ohULB/a99qV6bZJFfkVmzw2XON++aXW0GRPMGxrPAADI7 -C38ONWFAnYsC4aE2TZu6BAOwmUZSv4U0IY6uOZorSboIiUiD8BswSyX5nWlTLVLi -JlXudfdEb7C5UIJdO6uRUf+78RPNN/ZxVuVbLOOwE0Pcx7EWyM+4Wz1KNdumnT2n -rA7QQJ2frBLckNHLXh8HHmkk72a20DmFNrNZjj1sXpwBE+AqE7knZAozAF5dRVKX -4Jnh5qTaHDvobKIqwVt6yOX0knQp6UwT1hgmWtk8584hLoxuQbzr5yM3/QNiPMka -+aMGu1COS34ppIQfETt1C3Lv7tGcFXkadyV7JfVl0u8qAJjGzKgmrBcVkDBZdKT6 -rP3QAAwpcwdDaY0rXR6FDrHKZWZhvsSNjX9vXf6ThCG4/Wb07bMia1Y6UdtNWYAB -uDQkFJszng03tQP8P1p1J3XouzOb0XA3tkusOoRnMlR47oDb1TlUJKuJv7ztK3AI -5v7FyJO9TJbwhBWWno171KKGn7R1vciNYxWB1IqGamLQFKajTFKYVikn7mCdfDNe -zVgGcK9DAgoJXtclmjtKzUnqfW6ctJKrNKXkyEgkbWKGv33skwDSMZxakTpFCADt -dzG+RbDlZrwlSMhv+33vjV6arulEY32vj3Thbphb4v1JwOTdXNVwF5pC+pf7sUgx -ZAbEuVix4zu+AznCdRGPYjAHZgUe8ldp7JrQuVzlknBJ0Z+DedXcvL8RAs7rFedo -BO9FTqIolTqFZyBJeaDJcMTwf8qQdE71+n/r9oigKi/8UXSAZInnvgYMTUjgegqx -UyQJTEXRQYzUfCUHFpYqel1NwcsLO4JciDwb0ak1QUvIy0HulLrRKrRYUZ7Xy3U9 -XBjt8BPWKdNfN6qKWK+7tYzPNB2BLQT2Z+l0L+Ir9TaWxJTZdfRysYytAhTl3ZoL -99KlA7qIxOs7+DnuR8jdCAD64/v/xNl02ck3pBnnxjnaspe3xEz5D9s/a9Ob+2MU -pDp5b0ZwPJ0tpk6CguiVhi1gzRXzjABPrCbsBOXk2gQ+Pilk3jkugG+wqg4LOLzy -3AiDbD+GVmTL9gdhPNoNIjPBz1tkZEGdTVsUOWT5Kuom3KvbmzowdlTVb4x/kkRM -oOTfO1Uhna3b/pcCsMiHXkM3ZME+DFEy6Imh38742ljllECCDCUAvd4drbx4VaNU -rXyI+j3NP8j/vHhWCld0Bg/C5Xq9ioDOzyrvApTat2OZF2Z9K4nYYTTwniAHJWNk -hIPDKyIKeE4Y/WBrSfVQbdtTNi0fj+3RTtS1j4uSwMObCADf3miT0aX2pBxy7N6D -krhJwq5Usaeu3fVNOOm/HVQsouqJ0BYPX412YIDtL4iY/YibVdzVlNg2hQuPJYGx -hZF/isY+qpckLvdABbtreBDxoqtqBgdOI0EGXpBuJTq9iuh2kiKFWATEchCMND9p -wZK0jrVtQ7JnXwW4CRBnN0wV2LO8+VF6How9FSdVRykG4cRed7JB5U9B11PK3tay -v82+YnmTEr7YuMiJoXHt7+9nA1ftd/Au0IegwvmRO89MxvpGeS8GVZFdYLP5UQiH -bdrnFmLyQEsPPIM3yvFQCfb2W4WmxH3PycTrNDs7mone1L/pk/eMdg7QYF+E/QZD -1N1pfKeJAjYEGAEKACAWIQQBHXUy4S6x61I7K1L/Gb3KATgePQUCZNuPMQIbDAAK -CRD/Gb3KATgePW6sD/sHl+DskI2dz4Ym7ZKng39QHOKFdzoK9xtnLQSynqfrwzvz -UEzsqWNjOHONfzat9savfGBvWB6ULz3vQ4yZ731Y4Tdc4UmVo+79hIgeArNiLyou -p6tvcl+bKOv72xlrYn43lYy51Aj5TsU6TBGVvRETnmOWvzGzsQrhcZ+IA8p4JXoy -fiAM+reuc1lwWYZWS5n+XotskFS3L6BIXLPW1QIHxvcMkZAI8/VeloQej1z4t50i -7mAAXB/RwjXGCLnhFxhjWLHglrxpg6JpsuCkI7j5csYQuzhKrSWa3mxKv7rysRh3 -VpIkHfRRu/kyfOwM4vYCRsQH2lBl/JVgAn1koZKdnGomoq/Gw7+psuYNGX9+CQ5E -ShHGWGE2TUjCi88FDLDpOOmpKMoK19HbNGkeVeKXY80D5LhjSrOL61IY+3Hzyb0a -JhHA7dXAqMHak38Ps5sUrEtgDBjkhGtctCUGV9OsvlqAXwIDLB96W5LKJKw48osE -vcfquO0z3bMKyr0irCyBzn3sPd2nubFy/o1umF8efg2OqWybQ3PZLyMojQ9uu8Yb -SLM3+Q1vX/yAY8+KmaAvvUuTCS1dRWAM0s1g86AUCZ5LbMa0HwIGjCSA2nUw8BD8 -l3sVHJsSKOhB0XTUBVkIDBPdJcj/K6G8E0PGO19UQEMRis0/jnPPL9t8G63xQg== -=uAYd ------END PGP PRIVATE KEY BLOCK----- diff --git a/yezzey_test/pub.gpg b/yezzey_test/pub.gpg deleted file mode 100644 index 139122aea2a..00000000000 --- a/yezzey_test/pub.gpg +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBGTbjzEBEADOjtlqSaBe89Pg/orabZALZWi3lRfVqDaBjb/po5jOPlsefBSb -aa7hyAGhMfncmVaqZJXH1ikT2oHx430GcBXD8gjMZbil7U01DK8XBr6iSw/EPWpA -0jD2FdmPCG6vzF58KJ5tv2uPjZUShhYymIwDMZuQAzDMFgGP3dhDQNp4TUXX0AyH -vRGMEeckyjyXhheu1alJ1XNzT9s3dAq3LGFgHuUH4BEwSljpKFP+BARXTGq776wP -JXt7YaJLDB/RXgfMNOKZR18eUYEcckjziMkptBvbD6sG/h9B7ldeYCP91jDqR51Q -ls4eFf6C0dqoJ5ELv8GBW3BZWsnFGffGRA2a6kAk14eSWtcF7w0N5J7o4kPYJSPa -OXq+gIvBz9KeJgQKDu6DrHZYdZh5cCQKFHdLeKzaEsrUPvSOvjfTB8+ywb3QiJPt -/yKYfcZU41zB7dCfbbEFaKTGEvjRUm2XZH9maJvQ5fNUEmtXwQd05nBwN8aW+00d -w5Jy35aSwogCLrJuHMpwI56TRAU81d1R9RaTxNbcZUro8IdOW78BaB8RGrvGT+z1 -+TBPlJOpzJgAYJe6wZp8M4xk0z2y+6ia1X+yUvZXr51U8qiHM1wr1uBZx66EdFv+ -HQf3grEnW3sPCS3kF4Yn6Xm/xIWqbRlXf988RqZc6fKa201DhriEOhk9SQARAQAB -tCxSb290IFN1cGVydXNlciA8cm9vdEBoYW5kYm9vay53ZXN0YXJldGUuY29tPokC -TgQTAQoAOBYhBAEddTLhLrHrUjsrUv8ZvcoBOB49BQJk248xAhsDBQsJCAcCBhUK -CQgLAgQWAgMBAh4BAheAAAoJEP8ZvcoBOB49cZYQAJGLDODHkRVKlAQp7v2xECQ7 -2GYAHfcPh03jL4P7aRRj+lo+Dd60J08glCzqoUW74chacJMaJR6lywF5S9GlMPCC -4/oy1fFM5uZpTbRmhJtlmOqcZnlX1WXAJCMlaPdE37sFhWwQI8kZ8Qxb4zacdfcy -samiF+gqlpygvaSNWzC9ahwYc0gwelUeXIYZS3BUfPeQ7kerBxm9bBmPXz10pMxR -Kx9td3PvbcEOn6b80tSLwqL8FmmxHBJeoZul1DwthpIm+1LQNj+sggsg7Ih+Gccr -viuAKKnS99mx/Klznxis+uP36l8XtHHTSbRF8aWX6UPU+FrgZK1LU++2l21FO9/q -qyYCs42rU3uFrxgke0+55MqWf/uKTeOqKvu87tIsv4JNr9d9LWNomGicc5F9b596 -Sd1crfjOtDvlwWfsPkGqpxNe7aGRMNs+p31oGitYVT+9uplFUfVr9FFxVT1I4Ocj -B0KbTrFcui6NyvV1pl30ZZPQ8y8Udp6FD/jNt13+lgn159TXOV+l3DFSkFbJ9QQ4 -ZvCkca/sWNcgdKQeRM2Bo4rkA/QWxIZ+MDNPGr/bItFAqbl/capY05nuemsuDzlP -LGH+B1muTyjf+lbs/9cMtTqN+FZXlhHabF11oWzo/sP6O9F/cRUdske1wrfS6wha -MYNbI2TfXTDpDcK6Z8SjuQINBGTbjzEBEADoueEGCd+fQHjTkUYNtv4w6P/R2aZv -zVfAvnZmpsAiyiYxR+vPMUclvQ+17FKwivsj43B5UJK9zdRoZwwsXymo5Wc/imQ9 -eL6EGUZ414TgpC8ymfnqcuxn3Qa2kpgsuwxAHpUyb3z6/jAInqY+Qzz1I9CVzur1 -AV8LfWC2ttvlj/94OoIABOJUh1nqcQU0qA0MwoLRSZFj4PdFp4SU+lY7mVYMOV1K -M5n386t2D9NJ1G+Irv43N3I4zzFRzqVC6cbbqpEf9upDMeW4FgynhbeXvU6N/xiV -IBOTOIF0B3ClDUxXHisKTSjmdi4Z/nyyDFMivqohHjvvtnuVqdz261wt5JuW2SJ3 -7MBNbzE8U8bUgPqk1nQlfa21DsdABvdF80ORnEaKOoScnKNjSck43T/wlzPjoaR5 -TORp7Ssj9M5Ls4BKzXc1YxBXxOyoPL1UxP6lxMOPLVFp3ySte2tor3daUg9aFva8 -JImQIyvjXZuXZI6u1hR27T3fItiQSSurq9myXjOQg+sJu4R0lmBKfl3becd3YBGx -qPoPgplE/12DsDtGkR7M0sCHQUPAwt7+cmk19B130xchMAtzQLAn/HSwUGuPX6ru -MQti1KnOH33cLiL3JI5pYp84ZX1SN5IsFh8TG7GKwxDIAigPyGE04eepZWBxiont -HfDTKZT3ozb0zwARAQABiQI2BBgBCgAgFiEEAR11MuEusetSOytS/xm9ygE4Hj0F -AmTbjzECGwwACgkQ/xm9ygE4Hj1urA/7B5fg7JCNnc+GJu2Sp4N/UBzihXc6Cvcb -Zy0Esp6n68M781BM7KljYzhzjX82rfbGr3xgb1gelC8970OMme99WOE3XOFJlaPu -/YSIHgKzYi8qLqerb3Jfmyjr+9sZa2J+N5WMudQI+U7FOkwRlb0RE55jlr8xs7EK -4XGfiAPKeCV6Mn4gDPq3rnNZcFmGVkuZ/l6LbJBUty+gSFyz1tUCB8b3DJGQCPP1 -XpaEHo9c+LedIu5gAFwf0cI1xgi54RcYY1ix4Ja8aYOiabLgpCO4+XLGELs4Sq0l -mt5sSr+68rEYd1aSJB30Ubv5MnzsDOL2AkbEB9pQZfyVYAJ9ZKGSnZxqJqKvxsO/ -qbLmDRl/fgkOREoRxlhhNk1IwovPBQyw6TjpqSjKCtfR2zRpHlXil2PNA+S4Y0qz -i+tSGPtx88m9GiYRwO3VwKjB2pN/D7ObFKxLYAwY5IRrXLQlBlfTrL5agF8CAywf -eluSyiSsOPKLBL3H6rjtM92zCsq9Iqwsgc597D3dp7mxcv6NbphfHn4Njqlsm0Nz -2S8jKI0PbrvGG0izN/kNb1/8gGPPipmgL71LkwktXUVgDNLNYPOgFAmeS2zGtB8C -BowkgNp1MPAQ/Jd7FRybEijoQdF01AVZCAwT3SXI/yuhvBNDxjtfVEBDEYrNP45z -zy/bfBut8UI= -=x2ib ------END PGP PUBLIC KEY BLOCK----- diff --git a/yezzey_test/run_tests.sh b/yezzey_test/run_tests.sh deleted file mode 100755 index 68dff441c49..00000000000 --- a/yezzey_test/run_tests.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -set -ex - -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/id_rsa -sudo service ssh start -ssh -o StrictHostKeyChecking=no gpadmin@$(hostname) "echo 'Hello world'" - -sudo bash -c 'cat >> /etc/ld.so.conf <<-EOF -/usr/local/lib - -EOF' -sudo ldconfig - -sudo bash -c 'cat >> /etc/sysctl.conf <<-EOF -kernel.shmmax = 500000000 -kernel.shmmni = 4096 -kernel.shmall = 4000000000 -kernel.sem = 500 1024000 200 4096 -kernel.sysrq = 1 -kernel.core_uses_pid = 1 -kernel.msgmnb = 65536 -kernel.msgmax = 65536 -kernel.msgmni = 2048 -net.ipv4.tcp_syncookies = 1 -net.ipv4.ip_forward = 0 -net.ipv4.conf.default.accept_source_route = 0 -net.ipv4.tcp_tw_recycle = 1 -net.ipv4.tcp_max_syn_backlog = 4096 -net.ipv4.conf.all.arp_filter = 1 -net.ipv4.ip_local_port_range = 1025 65535 -net.core.netdev_max_backlog = 10000 -net.core.rmem_max = 2097152 -net.core.wmem_max = 2097152 -vm.overcommit_memory = 2 - -EOF' - -sudo bash -c 'cat >> /etc/security/limits.conf <<-EOF -* soft nofile 65536 -* hard nofile 65536 -* soft nproc 131072 -* hard nproc 131072 - -EOF' - -export GPHOME=/usr/local/gpdb -source $GPHOME/cloudberry-env.sh -ulimit -n 65536 -make destroy-demo-cluster && make create-demo-cluster -export USER=gpadmin -source gpAux/gpdemo/gpdemo-env.sh - -gpconfig -c shared_preload_libraries -v yezzey - -gpstop -a -i && gpstart -a - -createdb $USER - - -gpconfig -c yezzey.yproxy_socket -v "'/tmp/yproxy.sock'" -psql -c "ALTER SYSTEM SET yezzey.use_gpg_crypto TO false" -gpconfig -c yezzey.use_otm_feature -v "true" -gpconfig -c yezzey.use_gpg_crypto -v "false" - -gpstop -a -i && gpstart -a - -#run yproxy in daemon mode -/usr/bin/yproxy -c /tmp/yproxy.yaml -ldebug > yproxy.log 2>&1 & - -i=0 -while (! [ -S /tmp/yproxy.sock ]) && [ $i -lt 20 ]; do sleep 1; i=$(($i+1)) ; done - -cd gpcontrib/yezzey -make installcheck || (echo Yproxy logs; cat ../../yproxy.log; cat /home/gpadmin/gpcontrib/yezzey/regression.diffs && exit 1) diff --git a/yezzey_test/wal-g-conf.yaml b/yezzey_test/wal-g-conf.yaml deleted file mode 100644 index 8de5c4c2159..00000000000 --- a/yezzey_test/wal-g-conf.yaml +++ /dev/null @@ -1,12 +0,0 @@ -AWS_ACCESS_KEY_ID: "$AWS_ACCESS_KEY_ID" -AWS_SECRET_ACCESS_KEY: "$AWS_SECRET_ACCESS_KEY" -AWS_ENDPOINT: "$AWS_ENDPOINT" -AWS_S3_FORCE_PATH_STYLE: true - -WALG_COMPRESSION_METHOD: "brotli" -WALG_DELTA_MAX_STEPS: 6 -WALG_UPLOAD_CONCURRENCY: 10 -WALG_DISK_RATE_LIMIT: 41943040 -WALG_NETWORK_RATE_LIMIT: 10485760 -WALG_S3_PREFIX: "$WALG_S3_PREFIX" -WALG_PGP_KEY_PATH: "/home/gpadmin/yezzey_test/priv.gpg" diff --git a/yezzey_test/yproxy.conf b/yezzey_test/yproxy.conf deleted file mode 100644 index 855c3fce7d7..00000000000 --- a/yezzey_test/yproxy.conf +++ /dev/null @@ -1,25 +0,0 @@ -socket_path: "/tmp/yproxy.sock" -interconnect_socket_path: "/tmp/ic.sock" -log_level: debug - -storage: - access_key_id: "$AWS_ACCESS_KEY_ID" - secret_access_key: "$AWS_SECRET_ACCESS_KEY" - storage_endpoint: "$AWS_ENDPOINT" - storage_prefix: "" - storage_bucket: "gpyezzey" - storage_region: "us-west-2" - storage_type: "s3" - tablespace_map: - "pg_default": "gpyezzey" - "tab1": "gpyezzey2" - "tab2": "gpyezzey3" - -backup_storage: - access_key_id: "$AWS_ACCESS_KEY_ID" - secret_access_key: "$AWS_SECRET_ACCESS_KEY" - storage_endpoint: "$AWS_ENDPOINT" - storage_prefix: "" - storage_bucket: "gpyezzey" - storage_region: "us-west-2" - storage_type: "s3" From a9b3ddd0d427ee7f92405720ad6c9b2e15aa3b99 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 25 Feb 2026 13:36:13 +0300 Subject: [PATCH 010/167] Fix gpexpand not changing content-id for wal-g cmd When expanding a cluster, gpexpand copies the postgresql.conf file directly from the template segment (content 0). This causes issues for tools like wal-g which use a --content-id flag in archive_command and restore_command. Previously, new segments inherited --content-id=0 from the template. This caused them to push WAL segments to the wrong location, potentially overwriting segment 0's segments. This fix ensures the content ID in archive_command and restore_command is updated to match the new segment's ID during expansion. If the commands do not contain the --content-id flag, they remain unchanged. --- gpMgmt/bin/lib/gpconfigurenewsegment | 20 +- .../gpexpand_archive_restore_conf.out | 1047 +++++++++++++++++ .../isolation2_expandshrink_schedule | 3 +- .../sql/gpexpand_archive_restore_conf.sql | 48 + 4 files changed, 1116 insertions(+), 2 deletions(-) create mode 100644 src/test/isolation2/expected/gpexpand_archive_restore_conf.out create mode 100644 src/test/isolation2/sql/gpexpand_archive_restore_conf.sql diff --git a/gpMgmt/bin/lib/gpconfigurenewsegment b/gpMgmt/bin/lib/gpconfigurenewsegment index c37c70bf1fa..a372ad291ec 100755 --- a/gpMgmt/bin/lib/gpconfigurenewsegment +++ b/gpMgmt/bin/lib/gpconfigurenewsegment @@ -9,7 +9,7 @@ import shutil from optparse import Option, OptionGroup, OptionParser, OptionValueError, SUPPRESS_USAGE from gppylib.gpparseopts import OptParser, OptChecker -from gppylib.commands.gp import ModifyConfSetting, SegmentStart, SegmentStop +from gppylib.commands.gp import GpConfigHelper, ModifyConfSetting, SegmentStart, SegmentStop from gppylib.commands.pg import PgBaseBackup from gppylib.db import dbconn from gppylib.commands import unix @@ -268,6 +268,24 @@ class ConfExpSegCmd(Command): self.set_results(modifyConfCmd.get_results()) raise + # Update --content-id flag in wal-g for archive-restore GUCs if present + for guc in ['archive_command', 'restore_command']: + read_cmd = GpConfigHelper('Read %s' % guc, self.datadir, guc, getParameter=True) + read_cmd.run(validateAfter=True) + + val = read_cmd.get_value() + if not read_cmd.was_successful() or not val: + continue + + new_val = re.sub(r'(--content-id(?:=|\s+))-?\d+', r'\g<1>' + str(self.contentid), val) + if new_val != val: + write_cmd = GpConfigHelper('Update %s' % guc, self.datadir, guc, value=new_val) + try: + write_cmd.run(validateAfter=True) + except Exception: + self.set_results(write_cmd.get_results()) + raise + # We might need to stop the segment if the last setup failed past this point if os.path.exists('%s/postmaster.pid' % self.datadir): logger.info('%s/postmaster.pid exists. Stopping segment' % self.datadir) diff --git a/src/test/isolation2/expected/gpexpand_archive_restore_conf.out b/src/test/isolation2/expected/gpexpand_archive_restore_conf.out new file mode 100644 index 00000000000..08d97a49c47 --- /dev/null +++ b/src/test/isolation2/expected/gpexpand_archive_restore_conf.out @@ -0,0 +1,1047 @@ +-- Verify that gpexpand correctly updates segment-specific flags for wal-g in +-- archive_command and restore_command when initializing new segments. +-- +-- Previously, these GUCs were copied verbatim from the template segment +-- (content 0), causing new segments to invoke archiving with an incorrect +-- content-id (possible overwriting archive or restoring other's data). + +-- Cleanup any previous state +!\retcode yes | gpexpand -c; +-- start_ignore +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state + +System Expansion is used to add segments to an existing CBDB array. +gpexpand did not detect a System Expansion that is in progress. + +Before initiating a System Expansion, you need to provision and burn-in +the new hardware. Please be sure to run gpcheckperf to make sure the +new hardware is working properly. + +Please refer to the Admin Guide for more information. + +Would you like to initiate a new System Expansion Yy|Nn (default=N): +> +Enter a comma separated list of new hosts you want +to add to your array. Do not include interface hostnames. +**Enter a blank line to only add segments to existing hosts**[]: +> 20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[ERROR]:-gpexpand failed: You must be adding two or more hosts when expanding a system with mirroring enabled. + +Exiting... +20260225:10:13:28:029615 gpexpand:cdw:gpadmin-[INFO]:-Shutting down gpexpand... + +-- end_ignore +(exited with code 3) +!\retcode gpshrink -c; +-- start_ignore +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-Removing gpshrink schema +20260225:10:13:28:029633 gpshrink:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode rm -r /tmp/datadirs/; +-- start_ignore +rm: cannot remove '/tmp/datadirs/': No such file or directory + +-- end_ignore +(exited with code 1) + +-- Set GUCs on all segments, hardcode --content-id to 0 +!\retcode gpconfig -c restore_command -v '/bin/true'; +-- start_ignore +20260225:10:13:29:029665 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-c restore_command -v /bin/true' + +-- end_ignore +(exited with code 0) +!\retcode gpconfig -c archive_command -v 'wal-g seg wal-push %p --content-id=0'; +-- start_ignore +20260225:10:13:29:029714 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-c archive_command -v 'wal-g seg wal-push %p --content-id=0'' + +-- end_ignore +(exited with code 0) +!\retcode gpstop -u; +-- start_ignore +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Starting gpstop with args: -u +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Gathering information and validating the environment... +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Obtaining Cloudberry Coordinator catalog information +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Obtaining Segment details from coordinator... +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:29:029763 gpstop:cdw:gpadmin-[INFO]:-Signalling all postmaster processes to reload + +-- end_ignore +(exited with code 0) + +-- Prepare expansion configuration +!\retcode echo "localhost|localhost|7008|/tmp/datadirs/dbfast4/demoDataDir3|9|3|p localhost|localhost|7009|/tmp/datadirs/dbfast_mirror4/demoDataDir3|10|3|m" > /tmp/testexpand; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Expand +!\retcode gpexpand -i /tmp/testexpand; +-- start_ignore +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0 for dbid 2: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0 for dbid 5: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1 for dbid 3: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1 for dbid 6: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2 for dbid 4: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2 for dbid 7: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Heap checksum setting consistent across cluster +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[INFO]:-Syncing Apache Cloudberry extensions +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[ERROR]:-Syncing of Apache Cloudberry extensions has failed. +Traceback (most recent call last): + File "/usr/local/cloudberry-db/bin/gpexpand", line 1935, in sync_packages + operation.get_ret() + File "/usr/local/cloudberry-db/lib/python/gppylib/operations/__init__.py", line 64, in get_ret + if isinstance(self.ret, Exception): +AttributeError: 'SyncPackages' object has no attribute 'ret' +20260225:10:13:29:029792 gpexpand:cdw:gpadmin-[WARNING]:-Please run gppkg --clean after successful expansion. +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:13:30:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating segment template +20260225:10:13:33:029792 gpexpand:cdw:gpadmin-[INFO]:-Copying postgresql.conf from existing segment into template +20260225:10:13:33:029792 gpexpand:cdw:gpadmin-[INFO]:-Copying pg_hba.conf from existing segment into template +20260225:10:13:34:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating schema tar file +20260225:10:13:36:029792 gpexpand:cdw:gpadmin-[INFO]:-Distributing template tar file to new hosts +20260225:10:13:52:029792 gpexpand:cdw:gpadmin-[INFO]:-Configuring new segments (primary) +20260225:10:13:52:029792 gpexpand:cdw:gpadmin-[INFO]:-{'localhost': '/tmp/datadirs/dbfast4/demoDataDir3:7008:true:false:9:3::-1:'} +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-Starting to create new pg_hba.conf on primary segments +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-None of the reachable segments require update to pg_hba.conf +20260225:10:13:54:029792 gpexpand:cdw:gpadmin-[INFO]:-SegmentStart pg_ctl cmd is env GPSESSID=0000000000 GPERA=None $GPHOME/bin/pg_ctl -D /tmp/datadirs/dbfast4/demoDataDir3 -l /tmp/datadirs/dbfast4/demoDataDir3/log/startup.log -w -t 600 -o " -p 7008 -c gp_role=utility -M " start +20260225:10:13:55:029792 gpexpand:cdw:gpadmin-[INFO]:-Cleaning up temporary template files +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Cleaning up databases in new segments. +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-SegmentStart pg_ctl cmd is env GPSESSID=0000000000 GPERA=None $GPHOME/bin/pg_ctl -D /tmp/datadirs/dbfast4/demoDataDir3 -l /tmp/datadirs/dbfast4/demoDataDir3/log/startup.log -w -t 600 -o " -p 7008 -c gp_role=utility " start +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:13:56:029792 gpexpand:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Creating expansion schema +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database postgres +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database template1 +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database isolation2parallelretrcursor +20260225:10:13:57:029792 gpexpand:cdw:gpadmin-[INFO]:-Populating gpexpand.status_detail with data from database isolation2test +20260225:10:13:58:029792 gpexpand:cdw:gpadmin-[INFO]:-Starting new mirror segment synchronization +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-Initialization of the system expansion complete. +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-To begin table expansion onto the new segments +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-rerun gpexpand +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:08:029792 gpexpand:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpexpand -i /tmp/testexpand; +-- start_ignore +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:08:030678 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state +20260225:10:14:09:030678 gpexpand:cdw:gpadmin-[INFO]:-EXPANSION COMPLETED SUCCESSFULLY +20260225:10:14:09:030678 gpexpand:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) + +-- Get the new segment's datadir (content=3) +!\retcode psql -d postgres -Aqt -c "SELECT datadir FROM gp_segment_configuration WHERE content = 3 AND role = 'p'" > /tmp/new_segment_datadir; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Confirm that the --content-id flag within archive_command has been +-- updated to match the new segment's content. The restore_command +-- lacks --content-id flag and should be unchanged. +! grep "^archive_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +archive_command='wal-g seg wal-push %p --content-id=3' + +! grep "^restore_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +restore_command='/bin/true' + + +-- Cleanup +!\retcode gpconfig -r restore_command; +-- start_ignore +20260225:10:14:09:030725 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-r restore_command' + +-- end_ignore +(exited with code 0) +!\retcode gpconfig -r archive_command; +-- start_ignore +20260225:10:14:10:030829 gpconfig:cdw:gpadmin-[INFO]:-completed successfully with parameters '-r archive_command' + +-- end_ignore +(exited with code 0) +!\retcode gpstop -u; +-- start_ignore +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Starting gpstop with args: -u +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Gathering information and validating the environment... +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Obtaining Cloudberry Coordinator catalog information +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Obtaining Segment details from coordinator... +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:10:030933 gpstop:cdw:gpadmin-[INFO]:-Signalling all postmaster processes to reload + +-- end_ignore +(exited with code 0) + +!\retcode gpshrink -i /tmp/testexpand; +-- start_ignore +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast1/demoDataDir0 for dbid 2: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0 for dbid 5: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540553465864 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 4ab1899cbfd68b59268d02e0d5a1966c0ee3f4b463979bc5372053c2b14ece58 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1 for dbid 3: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1 for dbid 6: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540653109260 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: 91cf6a7a7826dd0c9f8ff23a43d226c7a1a72752a9b1544a78c7cfbe62970991 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2 for dbid 4: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:13:25 AM GMT +Latest checkpoint location: 0/C067FA8 +Latest checkpoint's REDO location: 0/C067F70 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:877 +Latest checkpoint's NextGxid: 9 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 876 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:13:25 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2 for dbid 7: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743540636610571 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:12:19 AM GMT +Latest checkpoint location: 0/C005578 +Latest checkpoint's REDO location: 0/C005578 +Latest checkpoint's REDO WAL file: 000000010000000000000003 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:872 +Latest checkpoint's NextGxid: 3 +Latest checkpoint's NextOID: 16392 +Latest checkpoint's NextRelfilenode: 12002 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 0 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:12:16 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/C005600 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: deb1a8b64d8bc3ddf938a6ec69c7c9785d8f32343c42e3878fe207fbee3ac992 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /tmp/datadirs/dbfast_mirror4/demoDataDir3 for dbid 10: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743519254633466 +Database cluster state: in archive recovery +pg_control last modified: Wed 25 Feb 2026 10:14:07 AM GMT +Latest checkpoint location: 0/1C000060 +Latest checkpoint's REDO location: 0/1C000028 +Latest checkpoint's REDO WAL file: 000000010000000000000007 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:966 +Latest checkpoint's NextGxid: 8196 +Latest checkpoint's NextOID: 33402 +Latest checkpoint's NextRelfilenode: 24576 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 966 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:14:02 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/1C000110 +Min recovery ending loc's timeline: 1 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: ec79bca6b7c99a5ea2ac78e41778e68c225231739152db9d7d59cc19b9096a29 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Successfully finished pg_controldata /tmp/datadirs/dbfast4/demoDataDir3 for dbid 9: +stdout: pg_control version number: 13000700 +Catalog version number: 302509031 +Database system identifier: 7610743519254633466 +Database cluster state: in production +pg_control last modified: Wed 25 Feb 2026 10:14:02 AM GMT +Latest checkpoint location: 0/1C000060 +Latest checkpoint's REDO location: 0/1C000028 +Latest checkpoint's REDO WAL file: 000000010000000000000007 +Latest checkpoint's TimeLineID: 1 +Latest checkpoint's PrevTimeLineID: 1 +Latest checkpoint's full_page_writes: on +Latest checkpoint's NextXID: 0:966 +Latest checkpoint's NextGxid: 8196 +Latest checkpoint's NextOID: 33402 +Latest checkpoint's NextRelfilenode: 24576 +Latest checkpoint's NextMultiXactId: 1 +Latest checkpoint's NextMultiOffset: 0 +Latest checkpoint's oldestXID: 810 +Latest checkpoint's oldestXID's DB: 13425 +Latest checkpoint's oldestActiveXID: 966 +Latest checkpoint's oldestMultiXid: 1 +Latest checkpoint's oldestMulti's DB: 1 +Latest checkpoint's oldestCommitTsXid:0 +Latest checkpoint's newestCommitTsXid:0 +Time of latest checkpoint: Wed 25 Feb 2026 10:14:02 AM GMT +Fake LSN counter for unlogged rels: 0/3E8 +Minimum recovery ending location: 0/0 +Min recovery ending loc's timeline: 0 +Backup start location: 0/0 +Backup end location: 0/0 +End-of-backup record required: no +wal_level setting: replica +wal_log_hints setting: off +max_connections setting: 750 +max_worker_processes setting: 14 +max_wal_senders setting: 10 +max_prepared_xacts setting: 250 +max_locks_per_xact setting: 128 +track_commit_timestamp setting: off +Maximum data alignment: 8 +Database block size: 32768 +Blocks per segment of large relation: 32768 +WAL block size: 32768 +Bytes per WAL segment: 67108864 +Maximum length of identifiers: 64 +Maximum columns in an index: 32 +Maximum size of a TOAST chunk: 8140 +Size of a large-object chunk: 8192 +Date/time type storage: 64-bit integers +Float8 argument passing: by value +Data page checksum version: 1 +Mock authentication nonce: ec79bca6b7c99a5ea2ac78e41778e68c225231739152db9d7d59cc19b9096a29 +File encryption method: + +stderr: +20260225:10:14:10:030997 gpshrink:cdw:gpadmin-[INFO]:-Heap checksum setting consistent across cluster +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:14:11:030997 gpshrink:cdw:gpadmin-[INFO]:-Creating shrink schema +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database postgres +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database template1 +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database isolation2parallelretrcursor +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Populating gpshrink.status_detail with data from database isolation2test +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Initialization of the system shrink complete. +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-To begin table shrink onto the new segments +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-rerun gpshrink +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-************************************************ +20260225:10:14:12:030997 gpshrink:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpshrink -i /tmp/testexpand; +-- start_ignore +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpexpand.status_detail +20260225:10:14:13:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpexpand.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpshrink.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpshrink.status_detail +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpexpand.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpexpand.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Shrinking postgres.gpshrink.status +20260225:10:14:14:031255 gpshrink:cdw:gpadmin-[INFO]:-Finished shrinking postgres.gpshrink.status +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-SHRINK COMPLETED SUCCESSFULLY +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-Locking catalog +20260225:10:14:18:031255 gpshrink:cdw:gpadmin-[INFO]:-Locked catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Unlocking catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Unlocked catalog +20260225:10:14:19:031255 gpshrink:cdw:gpadmin-[INFO]:-Exiting... + +-- end_ignore +(exited with code 0) + +!\retcode yes | gpexpand -c; +-- start_ignore +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Querying gpexpand schema for current expansion state + + +Do you want to dump the gpexpand.status_detail table to file? Yy|Nn (default=Y): +> 20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Dumping gpexpand.status_detail to /home/gpadmin/cloudberry/gpAux/gpdemo/datadirs/qddir/demoDataDir-1/gpexpand.status_detail +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Removing gpexpand schema +20260225:10:14:19:031525 gpexpand:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode gpshrink -c; +-- start_ignore +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 3.0.0-devel+dev.4.g01c21009c35 build dev' +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 14.4 (Apache Cloudberry 3.0.0-devel+dev.4.g01c21009c35 build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 11.4.0-1ubuntu1~22.04.2) 11.4.0, 64-bit compiled on Feb 25 2026 10:08:48 (with assert checking)' +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-Removing gpshrink schema +20260225:10:14:20:031554 gpshrink:cdw:gpadmin-[INFO]:-Cleanup Finished. exiting... + +-- end_ignore +(exited with code 0) +!\retcode rm -r /tmp/datadirs/; +-- start_ignore + +-- end_ignore +(exited with code 0) +!\retcode rm /tmp/new_segment_datadir; +-- start_ignore + +-- end_ignore +(exited with code 0) + diff --git a/src/test/isolation2/isolation2_expandshrink_schedule b/src/test/isolation2/isolation2_expandshrink_schedule index 3fd75a5eff0..89787bc1818 100644 --- a/src/test/isolation2/isolation2_expandshrink_schedule +++ b/src/test/isolation2/isolation2_expandshrink_schedule @@ -1,3 +1,4 @@ # Tests for gpexpand and gpshrink # Keep for single schedule due to redistribute all the table in all database -test: gpexpand_gpshrink \ No newline at end of file +test: gpexpand_gpshrink +test: gpexpand_archive_restore_conf diff --git a/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql b/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql new file mode 100644 index 00000000000..39cdf458f22 --- /dev/null +++ b/src/test/isolation2/sql/gpexpand_archive_restore_conf.sql @@ -0,0 +1,48 @@ +-- Verify that gpexpand correctly updates segment-specific flags for wal-g in +-- archive_command and restore_command when initializing new segments. +-- +-- Previously, these GUCs were copied verbatim from the template segment +-- (content 0), causing new segments to invoke archiving with an incorrect +-- content-id (possible overwriting archive or restoring other's data). + +-- Cleanup any previous state +!\retcode yes | gpexpand -c; +!\retcode gpshrink -c; +!\retcode rm -r /tmp/datadirs/; + +-- Set GUCs on all segments, hardcode --content-id to 0 +!\retcode gpconfig -c restore_command -v '/bin/true'; +!\retcode gpconfig -c archive_command -v 'wal-g seg wal-push %p --content-id=0'; +!\retcode gpstop -u; + +-- Prepare expansion configuration +!\retcode echo "localhost|localhost|7008|/tmp/datadirs/dbfast4/demoDataDir3|9|3|p +localhost|localhost|7009|/tmp/datadirs/dbfast_mirror4/demoDataDir3|10|3|m" > /tmp/testexpand; + +-- Expand +!\retcode gpexpand -i /tmp/testexpand; +!\retcode gpexpand -i /tmp/testexpand; + +-- Get the new segment's datadir (content=3) +!\retcode psql -d postgres -Aqt -c "SELECT datadir FROM gp_segment_configuration +WHERE content = 3 AND role = 'p'" > /tmp/new_segment_datadir; + +-- Confirm that the --content-id flag within archive_command has been +-- updated to match the new segment's content. The restore_command +-- lacks --content-id flag and should be unchanged. +! grep "^archive_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; +! grep "^restore_command" $(cat /tmp/new_segment_datadir)/postgresql.conf; + +-- Cleanup +!\retcode gpconfig -r restore_command; +!\retcode gpconfig -r archive_command; +!\retcode gpstop -u; + +!\retcode gpshrink -i /tmp/testexpand; +!\retcode gpshrink -i /tmp/testexpand; + +!\retcode yes | gpexpand -c; +!\retcode gpshrink -c; +!\retcode rm -r /tmp/datadirs/; +!\retcode rm /tmp/new_segment_datadir; + From 3a93184e79735888f33a5fe2bc5dbe0bfe855e26 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 21 Apr 2026 13:54:35 +0500 Subject: [PATCH 011/167] Bump yezzey. (#33) To the MWP cbdb version --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 4c6b5b83735..642726e074f 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 4c6b5b83735320dda01e042631a851336300a3ca +Subproject commit 642726e074f8a553af05e1eab8e5d460b45a93ca From b535d0371de9cefc4901d7a54fbd9a1aae974634 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Fri, 15 May 2026 17:35:47 +0300 Subject: [PATCH 012/167] Move yezzey forward to full support Cloudberry (#35) * Move yezzey forward to full support Cloudberry --- .github/workflows/yezzey-ci.yaml | 4 ++-- gpcontrib/yezzey | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index c1c41497a64..1d4402b0ad1 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -130,7 +130,7 @@ jobs: run: | set -ex pipefail # Download mc for Linux (amd64) - curl -O https://dl.min.io/client/mc/release/linux-amd64/mc + curl -fsSL -o mc https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc sudo mv mc /usr/local/bin/mc # Make mc available system-wide @@ -193,7 +193,7 @@ jobs: set -eo pipefail chmod +x "${SRC_DIR}"/gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && gpcontrib/yezzey/devops/scripts/prepare_test_yezzey.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR}/gpcontrib/yezzey && devops/scripts/prepare_test_yezzey.sh"; then echo "::error::Config yezzey failed" exit 1 fi diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 642726e074f..0d88f66a5fd 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 642726e074f8a553af05e1eab8e5d460b45a93ca +Subproject commit 0d88f66a5fd0dba82681eef5929529cb153cb325 From d0e4afeddc38b49ee83d0d1fd7e3a28e0cc8d84d Mon Sep 17 00:00:00 2001 From: reshke Date: Wed, 3 Jun 2026 11:49:10 +0500 Subject: [PATCH 013/167] Bump yezzey for PAX fix (#37) https://github.com/open-gpdb/yezzey/compare/0d88f66a5fd0dba82681eef5929529cb153cb325...2b5dcadd45b4183a4aa5ab976e50c97f0d4c7057 --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 0d88f66a5fd..2b5dcadd45b 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 0d88f66a5fd0dba82681eef5929529cb153cb325 +Subproject commit 2b5dcadd45b4183a4aa5ab976e50c97f0d4c7057 From f6b7727da597277a9a0e091c09d1d70e09770e90 Mon Sep 17 00:00:00 2001 From: Alena Rybakina <58230554+Alena0704@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:46:11 +0300 Subject: [PATCH 014/167] Allow non-superuser role to manage resource groups. Historically Yandex Greenplum allows non-superuser no managed resource groups. So, a regular non-superuser role allowed to run pg_resgroup_move_query(), and tune CPU/memory limits if granted with mdb_admin. Such feature was introduced as early as 6.22, see also gpdb commit 3ac99962. This commit introduces same feature for managed Cloudberry. To disallow altering predefined roles, fixed-OID hardening is used, reserving 8067 OID to be an mdb_admin role OID. We choose this (efficiently a catalog change) over complex bookkeeping what CREATEROLE can do and what is disallowed. We use Yandex managed predefined roles bootstrap util via auxiliary contrib extension, based on what Yandex Postgres fork does, see also pg-sharding/cpg repo. Co-authored-by: Andrey Borodin Co-authored-by: reshke --- .github/workflows/build-cloudberry-rocky8.yml | 1 + .github/workflows/build-cloudberry.yml | 1 + .github/workflows/build-deb-cloudberry.yml | 1 + contrib/Makefile | 1 + contrib/pg_aux_catalog/.gitignore | 6 + contrib/pg_aux_catalog/Makefile | 37 ++++++ contrib/pg_aux_catalog/README.md | 71 ++++++++++ .../expected/pg_aux_catalog.out | 46 +++++++ .../expected/resgroup_mdb_admin.out | 123 ++++++++++++++++++ .../isolation2/isolation2_schedule | 1 + .../isolation2/sql/resgroup_mdb_admin.sql | 91 +++++++++++++ .../pg_aux_catalog/pg_aux_catalog--1.0.sql | 10 ++ contrib/pg_aux_catalog/pg_aux_catalog.c | 88 +++++++++++++ contrib/pg_aux_catalog/pg_aux_catalog.control | 5 + contrib/pg_aux_catalog/sql/pg_aux_catalog.sql | 38 ++++++ pom.xml | 3 + src/backend/catalog/oid_dispatch.c | 50 ++++++- src/backend/commands/resgroupcmds.c | 51 +++++--- src/backend/utils/resgroup/resgroup_helper.c | 5 +- src/include/access/transam.h | 10 ++ src/include/catalog/oid_dispatch.h | 7 + src/include/utils/acl.h | 8 ++ 22 files changed, 631 insertions(+), 23 deletions(-) create mode 100644 contrib/pg_aux_catalog/.gitignore create mode 100644 contrib/pg_aux_catalog/Makefile create mode 100644 contrib/pg_aux_catalog/README.md create mode 100644 contrib/pg_aux_catalog/expected/pg_aux_catalog.out create mode 100644 contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out create mode 100644 contrib/pg_aux_catalog/isolation2/isolation2_schedule create mode 100644 contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog.c create mode 100644 contrib/pg_aux_catalog/pg_aux_catalog.control create mode 100644 contrib/pg_aux_catalog/sql/pg_aux_catalog.sql diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 2abf88060e3..c8068f098cb 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -311,6 +311,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index ca75f7b42e7..99bc67c99b7 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -304,6 +304,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 85d917b8ff0..fee69b073f7 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -243,6 +243,7 @@ jobs: "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", "contrib/passwordcheck:installcheck", + "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, diff --git a/contrib/Makefile b/contrib/Makefile index b14600e3557..01315b1f6f8 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -34,6 +34,7 @@ SUBDIRS = \ old_snapshot \ pageinspect \ passwordcheck \ + pg_aux_catalog \ postgres_fdw \ pg_buffercache \ pg_freespacemap \ diff --git a/contrib/pg_aux_catalog/.gitignore b/contrib/pg_aux_catalog/.gitignore new file mode 100644 index 00000000000..c4ec060cefb --- /dev/null +++ b/contrib/pg_aux_catalog/.gitignore @@ -0,0 +1,6 @@ +# Generated test output +/log/ +/results/ +/tmp_check/ +/isolation2/results/ +/isolation2/output_iso/ diff --git a/contrib/pg_aux_catalog/Makefile b/contrib/pg_aux_catalog/Makefile new file mode 100644 index 00000000000..439ecf31a76 --- /dev/null +++ b/contrib/pg_aux_catalog/Makefile @@ -0,0 +1,37 @@ +# contrib/pg_aux_catalog/Makefile + +MODULE_big = pg_aux_catalog +OBJS = \ + $(WIN32RES) \ + pg_aux_catalog.o + +EXTENSION = pg_aux_catalog +DATA = pg_aux_catalog--1.0.sql + +PGFILEDESC = "pg_aux_catalog - auxiliary catalog management" + +REGRESS = pg_aux_catalog +REGRESS_OPTS = --init-file=$(top_srcdir)/src/test/regress/init_file + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_aux_catalog +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +# Multi-session resource-group permission tests, run with the GPDB isolation2 +# harness. Requires a running cluster with resource groups enabled +# (gp_resource_manager=group); see installcheck-resgroup in +# src/test/isolation2. Not part of the default "installcheck". +installcheck-isolation2: install + $(pg_isolation2_regress_installcheck) \ + --init-file=$(top_builddir)/src/test/regress/init_file \ + --inputdir=$(srcdir)/isolation2 \ + --outputdir=isolation2 \ + --schedule=$(srcdir)/isolation2/isolation2_schedule + diff --git a/contrib/pg_aux_catalog/README.md b/contrib/pg_aux_catalog/README.md new file mode 100644 index 00000000000..a89535838ee --- /dev/null +++ b/contrib/pg_aux_catalog/README.md @@ -0,0 +1,71 @@ +# pg_aux_catalog + +Auxiliary catalog management for Apache Cloudberry. + +This extension provisions the **`mdb_admin`** privilege role, which lets a +non-superuser manage resource groups in managed-service deployments where the +client is never given superuser. + +## Background + +In Greenplum/Cloudberry only a superuser may `CREATE`/`ALTER`/`DROP` resource +groups or move a running query between groups with `pg_resgroup_move_query()`. +The server gates those four entry points on membership of `mdb_admin`, +identified by a **fixed OID (8067)** rather than by name, so the privilege is +recognised reliably across the coordinator and all segments. + +A fixed OID cannot be obtained from a plain `CREATE ROLE` (that assigns an +ordinary OID). This extension provides the one supported way to create the +role at OID 8067. + +## Functions + +### `pg_create_mdb_admin_role() returns oid` + +Creates the `mdb_admin` role with its fixed OID (8067). +Returns the OID of the created role (8067). Errors if a role with that OID or +the name `mdb_admin` already exists. The OID assignment is dispatched to the +segments, so the role has the same OID cluster-wide. + +## Usage + +```sql +CREATE EXTENSION pg_aux_catalog; + +-- Provision the role (the control plane does this once per cluster). +SELECT pg_create_mdb_admin_role(); + +-- Grant the capability to a tenant admin. +GRANT mdb_admin TO cloud_admin; + +-- cloud_admin can now manage resource groups without superuser: +SET ROLE cloud_admin; +CREATE RESOURCE GROUP rg_tenant WITH (concurrency = 4, cpu_max_percent = 20); +ALTER RESOURCE GROUP rg_tenant SET cpu_max_percent 30; +DROP RESOURCE GROUP rg_tenant; +``` + +`admin_group` and `system_group` remain superuser-only for `ALTER`/`DROP`: +they are infrastructure, not user-tunable groups. + +## Building and testing + +```sh +make -C contrib/pg_aux_catalog install +make -C contrib/pg_aux_catalog installcheck +``` + +`installcheck` runs a single-session regression test (role creation and the +resource-group permission gate). A multi-session isolation2 test covering the +dispatched / cross-session behaviour lives under `isolation2/` and is run +separately, against a cluster with resource groups enabled +(`gp_resource_manager=group`): + +```sh +make -C contrib/pg_aux_catalog installcheck-isolation2 +``` + +## Credits + +Based on [pg-sharding/cpg](https://github.com/pg-sharding/cpg) commit +`7b8c912`. Some tests are adapted from open-gpdb/gpdb commit `3ac99962ad2`. diff --git a/contrib/pg_aux_catalog/expected/pg_aux_catalog.out b/contrib/pg_aux_catalog/expected/pg_aux_catalog.out new file mode 100644 index 00000000000..32e52b9728d --- /dev/null +++ b/contrib/pg_aux_catalog/expected/pg_aux_catalog.out @@ -0,0 +1,46 @@ +-- Tests for the pg_aux_catalog extension: creation of the fixed-OID +-- mdb_admin role and the resource-group permission gate it enables. +CREATE EXTENSION pg_aux_catalog; +-- --------------------------------------------------------------------- +-- pg_create_mdb_admin_role() creates the mdb_admin role with its fixed OID. +-- --------------------------------------------------------------------- +SELECT pg_create_mdb_admin_role() AS mdb_admin_oid; + mdb_admin_oid +--------------- + 8067 +(1 row) + +-- The role exists with the fixed OID and is a non-login, non-superuser, +-- connection-limited role. +SELECT oid = 8067 AS has_fixed_oid, rolcanlogin, rolsuper, + rolcreaterole, rolcreatedb, rolconnlimit + FROM pg_authid WHERE rolname = 'mdb_admin'; + has_fixed_oid | rolcanlogin | rolsuper | rolcreaterole | rolcreatedb | rolconnlimit +---------------+-------------+----------+---------------+-------------+-------------- + t | f | f | f | f | 0 +(1 row) + +-- Creating it a second time is rejected. +SELECT pg_create_mdb_admin_role(); +ERROR: role with OID 8067 already exists +-- --------------------------------------------------------------------- +-- Resource-group permission gate: a role that is not a member of mdb_admin +-- is rejected on every entry point. These checks run before the "resource +-- group is enabled" check, so they are deterministic regardless of the +-- resource manager in use. +-- --------------------------------------------------------------------- +CREATE ROLE regress_rg_noadmin; +SET ROLE regress_rg_noadmin; +CREATE RESOURCE GROUP regress_rg_x WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +ALTER RESOURCE GROUP regress_rg_x SET cpu_max_percent 6; +ERROR: must be mdb_admin to alter resource groups +DROP RESOURCE GROUP regress_rg_x; +ERROR: must be mdb_admin to drop resource groups +RESET ROLE; +DROP ROLE regress_rg_noadmin; +-- --------------------------------------------------------------------- +-- Cleanup. +-- --------------------------------------------------------------------- +DROP ROLE mdb_admin; +DROP EXTENSION pg_aux_catalog; diff --git a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out new file mode 100644 index 00000000000..c8fd232faea --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out @@ -0,0 +1,123 @@ +-- Tests permission checks for the mdb_admin role with +-- resource groups enabled. + +-- start_matchsubs +-- m/ERROR: cannot find process: \d+/ +-- s/\d+/XXX/g +-- end_matchsubs + +DROP ROLE IF EXISTS role_rg_admin; +DROP +DROP ROLE IF EXISTS role_rg_noadmin; +DROP +DROP ROLE IF EXISTS mdb_admin; +DROP +-- start_ignore +DROP RESOURCE GROUP rg_perm_admin1; +DROP RESOURCE GROUP rg_perm_admin2; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_revoke2; +DROP RESOURCE GROUP rg_perm_test; +-- end_ignore + +-- --------------------------------------------------------------------- +-- Setup. The mdb_admin role is not predefined in the catalog; it is +-- created here the same way the control plane provisions it at runtime. +-- --------------------------------------------------------------------- +CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); +CREATE +CREATE ROLE mdb_admin; +CREATE +CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; +CREATE +CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; +CREATE +GRANT mdb_admin TO role_rg_admin; +GRANT + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +SET +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +ALTER +1: DROP RESOURCE GROUP rg_perm_admin1; +DROP + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +ERROR: must be superuser to alter resource group "admin_group" +1: DROP RESOURCE GROUP admin_group; +ERROR: must be superuser to drop resource group "admin_group" +1q: ... + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +SET +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +ERROR: must be mdb_admin to alter resource groups +2: DROP RESOURCE GROUP rg_perm_test; +ERROR: must be mdb_admin to drop resource groups +2q: ... + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: must be mdb_admin to move query +3: RESET ROLE; +RESET +3: SET ROLE role_rg_admin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: cannot find process: XXX +3q: ... + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +SET +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +5: REVOKE mdb_admin FROM role_rg_admin; +REVOKE +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +4: DROP RESOURCE GROUP rg_perm_revoke1; +ERROR: must be mdb_admin to drop resource groups +4q: ... +5q: ... + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +RESET +DROP ROLE role_rg_admin; +DROP +DROP ROLE role_rg_noadmin; +DROP +DROP ROLE mdb_admin; +DROP +DROP RESOURCE GROUP rg_perm_revoke1; +DROP +DROP RESOURCE GROUP rg_perm_test; +DROP diff --git a/contrib/pg_aux_catalog/isolation2/isolation2_schedule b/contrib/pg_aux_catalog/isolation2/isolation2_schedule new file mode 100644 index 00000000000..73b2a8a95a5 --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/isolation2_schedule @@ -0,0 +1 @@ +test: resgroup_mdb_admin diff --git a/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql b/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql new file mode 100644 index 00000000000..1b7ea19fb3e --- /dev/null +++ b/contrib/pg_aux_catalog/isolation2/sql/resgroup_mdb_admin.sql @@ -0,0 +1,91 @@ +-- Tests permission checks for the mdb_admin role with +-- resource groups enabled. + +-- start_matchsubs +-- m/ERROR: cannot find process: \d+/ +-- s/\d+/XXX/g +-- end_matchsubs + +DROP ROLE IF EXISTS role_rg_admin; +DROP ROLE IF EXISTS role_rg_noadmin; +DROP ROLE IF EXISTS mdb_admin; +-- start_ignore +DROP RESOURCE GROUP rg_perm_admin1; +DROP RESOURCE GROUP rg_perm_admin2; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_revoke2; +DROP RESOURCE GROUP rg_perm_test; +-- end_ignore + +-- --------------------------------------------------------------------- +-- Setup. mdb_admin is identified by its fixed OID, so it must be created +-- through contrib/pg_aux_catalog (a plain CREATE ROLE would assign a +-- different OID and the permission checks would not recognise its members). +-- --------------------------------------------------------------------- +CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); +CREATE EXTENSION IF NOT EXISTS pg_aux_catalog; +SELECT pg_create_mdb_admin_role(); +CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; +CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; +GRANT mdb_admin TO role_rg_admin; + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +1: DROP RESOURCE GROUP rg_perm_admin1; + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +1: DROP RESOURCE GROUP admin_group; +1q: + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +2: DROP RESOURCE GROUP rg_perm_test; +2q: + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +3: RESET ROLE; +3: SET ROLE role_rg_admin; +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +3q: + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +5: REVOKE mdb_admin FROM role_rg_admin; +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +4: DROP RESOURCE GROUP rg_perm_revoke1; +4q: +5q: + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +DROP ROLE role_rg_admin; +DROP ROLE role_rg_noadmin; +DROP ROLE mdb_admin; +DROP RESOURCE GROUP rg_perm_revoke1; +DROP RESOURCE GROUP rg_perm_test; diff --git a/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql b/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql new file mode 100644 index 00000000000..a1e1b00fcce --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql @@ -0,0 +1,10 @@ +/* contrib/pg_aux_catalog/pg_aux_catalog--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_aux_catalog" to load this file. \quit + +-- Create the mdb_admin role with fixed OID 8067 +CREATE FUNCTION pg_create_mdb_admin_role() +RETURNS OID +AS 'MODULE_PATHNAME', 'pg_create_mdb_admin_role' +LANGUAGE C PARALLEL SAFE STRICT; diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.c b/contrib/pg_aux_catalog/pg_aux_catalog.c new file mode 100644 index 00000000000..91685561cca --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog.c @@ -0,0 +1,88 @@ +/*------------------------------------------------------------------------- + * + * pg_aux_catalog.c + * Extension for auxiliary catalog management + * + * contrib/pg_aux_catalog/pg_aux_catalog.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/oid_dispatch.h" +#include "catalog/pg_authid.h" +#include "commands/user.h" +#include "fmgr.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/syscache.h" + +PG_MODULE_MAGIC; + +/* Name of the mdb_admin role; its OID is MDB_ADMIN_ROLEID (see acl.h). */ +#define MDB_ADMIN_ROLE_NAME "mdb_admin" + +PG_FUNCTION_INFO_V1(pg_create_mdb_admin_role); + +/* + * Create the mdb_admin role with its fixed OID (MDB_ADMIN_ROLEID, 8067). + * + * The core privilege checks identify mdb_admin by this fixed OID (see acl.c + * and resgroupcmds.c), so the role must always be created with it. On a + * Cloudberry cluster the OID is dispatched to the segments so the role ends + * up with the same OID everywhere. Returns the new role's OID. + */ +Datum +pg_create_mdb_admin_role(PG_FUNCTION_ARGS) +{ + CreateRoleStmt stmt; + List *options = NIL; + Oid roleid; + + /* + * Only a superuser may establish the mdb_admin privilege role. Otherwise + * a CREATEROLE user could drop mdb_admin and re-create it (CreateRole only + * requires CREATEROLE), taking ownership of the fixed-OID role and + * granting the capability to itself. + */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to create the mdb_admin role"))); + + /* Check if a role with the fixed OID already exists. */ + if (SearchSysCacheExists1(AUTHOID, ObjectIdGetDatum(MDB_ADMIN_ROLEID))) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("role with OID %u already exists", MDB_ADMIN_ROLEID))); + + /* Check if a role named "mdb_admin" already exists. */ + if (SearchSysCacheExists1(AUTHNAME, CStringGetDatum(MDB_ADMIN_ROLE_NAME))) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("role \"%s\" already exists", MDB_ADMIN_ROLE_NAME))); + + /* Build options for CreateRole: connection limit = 0. */ + options = list_make1(makeDefElem("connectionlimit", + (Node *) makeInteger(0), -1)); + + /* Prepare the CreateRoleStmt. */ + memset(&stmt, 0, sizeof(stmt)); + stmt.type = T_CreateRoleStmt; + stmt.stmt_type = ROLESTMT_ROLE; + stmt.role = MDB_ADMIN_ROLE_NAME; + stmt.options = options; + + /* + * Request the fixed OID for the role. GetNewOidForAuthId() consumes and + * clears this override. + */ + next_aux_pg_authid_oid = MDB_ADMIN_ROLEID; + + roleid = CreateRole(NULL, &stmt); + + PG_RETURN_OID(roleid); +} diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.control b/contrib/pg_aux_catalog/pg_aux_catalog.control new file mode 100644 index 00000000000..aff4205f7eb --- /dev/null +++ b/contrib/pg_aux_catalog/pg_aux_catalog.control @@ -0,0 +1,5 @@ +# pg_aux_catalog extension +comment = 'auxiliary catalog management (mdb_admin role creation)' +default_version = '1.0' +module_pathname = '$libdir/pg_aux_catalog' +relocatable = false diff --git a/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql b/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql new file mode 100644 index 00000000000..aa16ffe4e13 --- /dev/null +++ b/contrib/pg_aux_catalog/sql/pg_aux_catalog.sql @@ -0,0 +1,38 @@ +-- Tests for the pg_aux_catalog extension: creation of the fixed-OID +-- mdb_admin role and the resource-group permission gate it enables. + +CREATE EXTENSION pg_aux_catalog; + +-- --------------------------------------------------------------------- +-- pg_create_mdb_admin_role() creates the mdb_admin role with its fixed OID. +-- --------------------------------------------------------------------- +SELECT pg_create_mdb_admin_role() AS mdb_admin_oid; + +-- The role exists with the fixed OID and is a non-login, non-superuser, +-- connection-limited role. +SELECT oid = 8067 AS has_fixed_oid, rolcanlogin, rolsuper, + rolcreaterole, rolcreatedb, rolconnlimit + FROM pg_authid WHERE rolname = 'mdb_admin'; + +-- Creating it a second time is rejected. +SELECT pg_create_mdb_admin_role(); + +-- --------------------------------------------------------------------- +-- Resource-group permission gate: a role that is not a member of mdb_admin +-- is rejected on every entry point. These checks run before the "resource +-- group is enabled" check, so they are deterministic regardless of the +-- resource manager in use. +-- --------------------------------------------------------------------- +CREATE ROLE regress_rg_noadmin; +SET ROLE regress_rg_noadmin; +CREATE RESOURCE GROUP regress_rg_x WITH (concurrency=1, cpu_max_percent=5); +ALTER RESOURCE GROUP regress_rg_x SET cpu_max_percent 6; +DROP RESOURCE GROUP regress_rg_x; +RESET ROLE; +DROP ROLE regress_rg_noadmin; + +-- --------------------------------------------------------------------- +-- Cleanup. +-- --------------------------------------------------------------------- +DROP ROLE mdb_admin; +DROP EXTENSION pg_aux_catalog; diff --git a/pom.xml b/pom.xml index 0e000093399..6eaa095fa37 100644 --- a/pom.xml +++ b/pom.xml @@ -352,6 +352,9 @@ code or new licensing patterns. contrib/indexscan/indexscan.c contrib/indexscan/indexscan.sql.in + contrib/pg_aux_catalog/pg_aux_catalog.c + contrib/pg_aux_catalog/isolation2/isolation2_schedule + contrib/file_fdw/init_file contrib/file_fdw/data/** diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index 6f39a07857e..eaa2e099876 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -156,6 +156,18 @@ static MemoryContext oids_context = NULL; static bool preserve_oids_on_commit = false; +/* + * OID to assign to the next auxiliary pg_authid role created through + * GetNewOidForAuthId(), or InvalidOid for the normal allocation path. + * + * This is the GPDB analogue of upstream PostgreSQL's + * binary_upgrade_next_pg_authid_oid (which is disabled here in favour of the + * generic OID pre-assignment machinery). It lets contrib/pg_aux_catalog + * create roles such as mdb_admin with a fixed, well-known OID. It is reset + * to InvalidOid as soon as it is consumed. + */ +Oid next_aux_pg_authid_oid = InvalidOid; + /* * These will be used by the schema restoration process during binary upgrade, * so any new object must not use any Oid in this structure or else there will @@ -423,6 +435,7 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, OidAssignment *searchkey) { Oid oid; + Oid forcedOid = searchkey->oid; searchkey->catalog = RelationGetRelid(relation); @@ -461,8 +474,18 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, { MemoryContext oldcontext; - /* Assign a new oid, and memorize it in the list of OIDs to dispatch */ - oid = GetNewOidWithIndex(relation, indexId, oidcolumn); + /* + * Assign a new oid, and memorize it in the list of OIDs to dispatch. + * + * A caller may request a fixed, well-known OID by passing it in + * searchkey->oid (e.g. the mdb_admin auxiliary role, see + * GetNewOidForAuthId()). In that case use the requested OID instead + * of allocating a fresh one, but still record it for dispatch so the + * QEs end up with the same OID. + */ + oid = OidIsValid(forcedOid) + ? forcedOid + : GetNewOidWithIndex(relation, indexId, oidcolumn); oldcontext = MemoryContextSwitchTo(get_oids_context()); searchkey->oid = oid; @@ -479,7 +502,9 @@ GetNewOrPreassignedOid(Relation relation, Oid indexId, AttrNumber oidcolumn, } else { - oid = GetNewOidWithIndex(relation, indexId, oidcolumn); + oid = OidIsValid(forcedOid) + ? forcedOid + : GetNewOidWithIndex(relation, indexId, oidcolumn); } return oid; @@ -572,6 +597,25 @@ GetNewOidForAuthId(Relation relation, Oid indexId, AttrNumber oidcolumn, memset(&key, 0, sizeof(OidAssignment)); key.type = T_OidAssignment; key.objname = rolname; + + /* + * Allow auxiliary roles (such as mdb_admin, see contrib/pg_aux_catalog) + * to be created with a fixed, well-known OID. The OID is supplied through + * the next_aux_pg_authid_oid override, mirroring how upstream PostgreSQL + * assigns role OIDs during binary upgrade. We only honor it for OIDs in + * the auxiliary range to avoid clashing with normal OID allocation, and + * reset it immediately so it affects a single role only. + */ + if (OidIsValid(next_aux_pg_authid_oid)) + { + if (!IsAuxOid(next_aux_pg_authid_oid)) + elog(ERROR, "pre-assigned auxiliary role OID %u is out of the auxiliary OID range", + next_aux_pg_authid_oid); + + key.oid = next_aux_pg_authid_oid; + next_aux_pg_authid_oid = InvalidOid; + } + return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); } diff --git a/src/backend/commands/resgroupcmds.c b/src/backend/commands/resgroupcmds.c index 384675edb7f..b746a3db49b 100644 --- a/src/backend/commands/resgroupcmds.c +++ b/src/backend/commands/resgroupcmds.c @@ -34,6 +34,7 @@ #include "commands/resgroupcmds.h" #include "miscadmin.h" #include "nodes/pg_list.h" +#include "utils/acl.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/fmgroids.h" @@ -103,11 +104,11 @@ CreateResourceGroup(CreateResourceGroupStmt *stmt) int nResGroups; MemoryContext oldContext; - /* Permission check - only superuser can create groups. */ - if (!superuser()) + /* Permission check - only superuser or mdb_admin can create groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to create resource groups"))); + errmsg("must be mdb_admin to create resource groups"))); /* * Check for an illegal name ('none' is used to signify no group in ALTER ROLE). @@ -269,11 +270,11 @@ DropResourceGroup(DropResourceGroupStmt *stmt) Oid groupid; ResourceGroupCallbackContext *callbackCtx; - /* Permission check - only superuser can drop resource groups. */ - if (!superuser()) + /* Permission check - only superuser or mdb_admin can drop resource groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to drop resource groups"))); + errmsg("must be mdb_admin to drop resource groups"))); /* * Check the pg_resgroup relation to be certain the resource group already @@ -302,6 +303,13 @@ DropResourceGroup(DropResourceGroupStmt *stmt) */ groupid = ((Form_pg_resgroup) GETSTRUCT(tuple))->oid; + /* Permission check - only superuser can drop the admin/system resource groups. */ + if (!superuser() && (groupid == ADMINRESGROUP_OID || groupid == SYSTEMRESGROUP_OID)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to drop resource group \"%s\"", + stmt->name))); + /* cannot DROP default resource groups */ if (groupid == DEFAULTRESGROUP_OID || groupid == ADMINRESGROUP_OID @@ -375,11 +383,24 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) ResourceGroupCallbackContext *callbackCtx; MemoryContext oldContext; - /* Permission check - only superuser can alter resource groups. */ - if (!superuser()) + /* Permission check - only mdb_admin can alter resource groups. */ + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be mdb_admin to alter resource groups"))); + + /* + * Check the pg_resgroup relation to be certain the resource group already + * exists. + */ + groupid = get_resgroup_oid(stmt->name, false); + + /* Permission check - only superuser can alter the admin/system resource groups. */ + if (!superuser() && (groupid == ADMINRESGROUP_OID || groupid == SYSTEMRESGROUP_OID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("must be superuser to alter resource groups"))); + errmsg("must be superuser to alter resource group \"%s\"", + stmt->name))); /* Currently we only support to ALTER one limit at one time */ Assert(list_length(stmt->options) == 1); @@ -406,12 +427,6 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) checkResgroupCapLimit(limitType, value); } - /* - * Check the pg_resgroup relation to be certain the resource group already - * exists. - */ - groupid = get_resgroup_oid(stmt->name, false); - if (limitType == RESGROUP_LIMIT_TYPE_CONCURRENCY && value == 0 && groupid == ADMINRESGROUP_OID) @@ -500,7 +515,7 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) RESGROUP_DEFAULT_CPU_WEIGHT, ""); updateResgroupCapabilityEntry(pg_resgroupcapability_rel, - groupid, RESGROUP_LIMIT_TYPE_CPUSET, + groupid, RESGROUP_LIMIT_TYPE_CPUSET, 0, caps.cpuset); } else if (limitType == RESGROUP_LIMIT_TYPE_CPU) @@ -1007,7 +1022,7 @@ parseStmtOptions(CreateResourceGroupStmt *stmt, ResGroupCaps *caps) else mask |= 1 << type; - if (type == RESGROUP_LIMIT_TYPE_CPUSET) + if (type == RESGROUP_LIMIT_TYPE_CPUSET) { const char *cpuset = defGetString(defel); strlcpy(caps->cpuset, cpuset, sizeof(caps->cpuset)); @@ -1611,7 +1626,7 @@ checkCpuSetByRole(const char *cpuset) * ex: * cpuset = "1;4" * then we should assign '1' to corrdinator and '4' to segment - * + * * cpuset = "1" * assign '1' to both coordinator and segment */ diff --git a/src/backend/utils/resgroup/resgroup_helper.c b/src/backend/utils/resgroup/resgroup_helper.c index 00aaded168d..5acca0be5e7 100644 --- a/src/backend/utils/resgroup/resgroup_helper.c +++ b/src/backend/utils/resgroup/resgroup_helper.c @@ -21,6 +21,7 @@ #include "cdb/cdbvars.h" #include "commands/resgroupcmds.h" #include "storage/procarray.h" +#include "utils/acl.h" #include "utils/builtins.h" #include "utils/datetime.h" #include "utils/resgroup.h" @@ -464,10 +465,10 @@ pg_resgroup_move_query(PG_FUNCTION_ARGS) (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("resource group is not enabled")))); - if (!superuser()) + if (!is_member_of_role(GetUserId(), MDB_ADMIN_ROLEID)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - (errmsg("must be superuser to move query")))); + (errmsg("must be mdb_admin to move query")))); if (Gp_role == GP_ROLE_DISPATCH) { diff --git a/src/include/access/transam.h b/src/include/access/transam.h index 687799bec9f..a1bf330b762 100644 --- a/src/include/access/transam.h +++ b/src/include/access/transam.h @@ -208,6 +208,16 @@ FullTransactionIdAdvance(FullTransactionId *dest) #define FirstBinaryUpgradeReservedObjectId 9000 #define LastBinaryUpgradeReservedObjectId 9100 +/* + * Reserve a block of OIDs for auxiliary catalog objects (such as the + * mdb_admin role created by contrib/pg_aux_catalog). These need fixed, + * well-known OIDs that are stable across clusters, so they live in their own + * range below FirstBinaryUpgradeReservedObjectId. + */ +#define FirstAuxObjectId 8000 +#define LastAuxObjectId 9000 +#define IsAuxOid(oid) ((oid) >= FirstAuxObjectId && (oid) < LastAuxObjectId) + /* * VariableCache is a data structure in shared memory that is used to track * OID and XID assignment state. For largely historical reasons, there is diff --git a/src/include/catalog/oid_dispatch.h b/src/include/catalog/oid_dispatch.h index fbb7a14f59e..d9c543bee5d 100644 --- a/src/include/catalog/oid_dispatch.h +++ b/src/include/catalog/oid_dispatch.h @@ -16,6 +16,13 @@ #include "utils/relcache.h" #include "access/htup.h" +/* + * OID to assign to the next auxiliary pg_authid role created through + * GetNewOidForAuthId(), or InvalidOid for normal allocation. Set by + * contrib/pg_aux_catalog to create roles such as mdb_admin with a fixed OID. + */ +extern PGDLLIMPORT Oid next_aux_pg_authid_oid; + /* Functions used in master */ extern List *GetAssignedOidsForDispatch(void); diff --git a/src/include/utils/acl.h b/src/include/utils/acl.h index 49068f04b2f..e3949dd8769 100644 --- a/src/include/utils/acl.h +++ b/src/include/utils/acl.h @@ -213,6 +213,14 @@ extern bool is_member_of_role_nosuper(Oid member, Oid role); extern bool is_admin_of_role(Oid member, Oid role); // -- non-upstream patch begin +/* + * Fixed, well-known OID of the mdb_admin role. The role is created by + * contrib/pg_aux_catalog (pg_create_mdb_admin_role()) with this OID, and the + * resource-group permission checks identify it by OID. It lives in the + * auxiliary OID range (see IsAuxOid()). + */ +#define MDB_ADMIN_ROLEID 8067 + extern bool mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId); extern void check_mdb_admin_is_member_of_role(Oid member, Oid role); From 1aeb3b3e3660106cd674a7e11bce3c72d266ccd1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:22:35 +0000 Subject: [PATCH 015/167] Fix duplicate mdb_admin_allow_bypass_owner_checks definition in acl.c --- src/backend/utils/adt/acl.c | 42 ------------------------------------- 1 file changed, 42 deletions(-) diff --git a/src/backend/utils/adt/acl.c b/src/backend/utils/adt/acl.c index 1baf148f987..906480c5137 100644 --- a/src/backend/utils/adt/acl.c +++ b/src/backend/utils/adt/acl.c @@ -5129,48 +5129,6 @@ mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) // -- non-upstream patch end -// -- non-upstream patch begin -/* - * Is userId allowed to bypass ownership check - * and tranfer onwership to ownerId role? - */ -bool -mdb_admin_allow_bypass_owner_checks(Oid userId, Oid ownerId) -{ - Oid mdb_admin_roleoid; - /* - * Never allow nobody to grant objects to - * superusers. - * This can result in various CVE. - * For paranoic reasons, check this even before - * membership of mdb_admin role. - */ - if (superuser_arg(ownerId)) { - return false; - } - - mdb_admin_roleoid = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); - /* Is userId actually member of mdb admin? */ - if (!is_member_of_role(userId, mdb_admin_roleoid)) { - /* if no, disallow. */ - return false; - } - - /* - * Now, we need to check if ownerId - * is some dangerous role to trasfer membership to. - * - * For now, we check that ownerId does not have - * priviledge to execute server program or/and - * read/write server files, or/and pg read/write all data - */ - - /* All checks passed, hope will not be hacked here (again) */ - return !has_privs_of_unwanted_system_role(ownerId); -} - -// -- non-upstream patch end - /* * Is member a member of role (directly or indirectly)? * From a7895af766272ac4e074a89c351c7004a03be21a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:27:59 +0000 Subject: [PATCH 016/167] Address review comments: fix includes, configure options, pipefail, license headers, expected output --- .github/workflows/yezzey-ci.yaml | 4 +- configure.ac | 14 +++ .../expected/resgroup_mdb_admin.out | 103 +++++++++++++++++- .../isolation2/isolation2_schedule | 17 +++ contrib/pg_aux_catalog/pg_aux_catalog.c | 17 +++ gpcontrib/Makefile | 4 - pom.xml | 3 - src/backend/catalog/oid_dispatch.c | 1 + 8 files changed, 150 insertions(+), 13 deletions(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index 1d4402b0ad1..9913a6196c7 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -128,7 +128,7 @@ jobs: - name: Install MinIO Client (mc) run: | - set -ex pipefail + set -exo pipefail # Download mc for Linux (amd64) curl -fsSL -o mc https://dl.min.io/client/mc/release/linux-amd64/mc chmod +x mc @@ -136,7 +136,7 @@ jobs: - name: Configure MinIO service run: | - set -ex pipefail + set -exo pipefail # Add the MinIO service as an "alias" in mc (name it "minio-ci") mc alias set minio-ci http://minio:9000 some_key some_key diff --git a/configure.ac b/configure.ac index 89876e69d4f..308e0872f07 100644 --- a/configure.ac +++ b/configure.ac @@ -1380,6 +1380,20 @@ PGAC_ARG_BOOL(with, yezzey, no, [build with Yezzey extension]) AC_SUBST(with_yezzey) +# +# diskquota +# +PGAC_ARG_BOOL(with, diskquota, yes, + [build with diskquota extension]) +AC_SUBST(with_diskquota) + +# +# gp_stats_collector +# +PGAC_ARG_BOOL(with, gp_stats_collector, yes, + [build with gp_stats_collector extension]) +AC_SUBST(with_gp_stats_collector) + # # Realtime library # diff --git a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out index c8fd232faea..7d8ed663485 100644 --- a/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out +++ b/contrib/pg_aux_catalog/isolation2/expected/resgroup_mdb_admin.out @@ -21,13 +21,20 @@ DROP RESOURCE GROUP rg_perm_test; -- end_ignore -- --------------------------------------------------------------------- --- Setup. The mdb_admin role is not predefined in the catalog; it is --- created here the same way the control plane provisions it at runtime. +-- Setup. mdb_admin is identified by its fixed OID, so it must be created +-- through contrib/pg_aux_catalog (a plain CREATE ROLE would assign a +-- different OID and the permission checks would not recognise its members). -- --------------------------------------------------------------------- CREATE RESOURCE GROUP rg_perm_test WITH (concurrency=2, cpu_max_percent=10); CREATE -CREATE ROLE mdb_admin; -CREATE +CREATE EXTENSION IF NOT EXISTS pg_aux_catalog; +CREATE EXTENSION +SELECT pg_create_mdb_admin_role(); + pg_create_mdb_admin_role +-------------------------- + 8067 +(1 row) + CREATE ROLE role_rg_admin RESOURCE GROUP rg_perm_test; CREATE CREATE ROLE role_rg_noadmin RESOURCE GROUP rg_perm_test; @@ -121,3 +128,91 @@ DROP RESOURCE GROUP rg_perm_revoke1; DROP DROP RESOURCE GROUP rg_perm_test; DROP + + +-- --------------------------------------------------------------------- +-- 1. Member of mdb_admin can CREATE/ALTER/DROP resource groups +-- (statements are dispatched to segments). +-- --------------------------------------------------------------------- +1: SET ROLE role_rg_admin; +SET +1: CREATE RESOURCE GROUP rg_perm_admin1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +1: ALTER RESOURCE GROUP rg_perm_admin1 SET cpu_max_percent 6; +ALTER +1: DROP RESOURCE GROUP rg_perm_admin1; +DROP + +-- 2. Even a member cannot ALTER or DROP the system admin_group. +1: ALTER RESOURCE GROUP admin_group SET cpu_max_percent 99; +ERROR: must be superuser to alter resource group "admin_group" +1: DROP RESOURCE GROUP admin_group; +ERROR: must be superuser to drop resource group "admin_group" +1q: ... + +-- --------------------------------------------------------------------- +-- 3. A non-member is rejected on every entry point. +-- --------------------------------------------------------------------- +2: SET ROLE role_rg_noadmin; +SET +2: CREATE RESOURCE GROUP rg_perm_admin2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +2: ALTER RESOURCE GROUP rg_perm_test SET cpu_max_percent 7; +ERROR: must be mdb_admin to alter resource groups +2: DROP RESOURCE GROUP rg_perm_test; +ERROR: must be mdb_admin to drop resource groups +2q: ... + +-- --------------------------------------------------------------------- +-- 4. pg_resgroup_move_query() honours the same permission check. +-- The first call (non-member) must fail with "must be mdb_admin". +-- The second call (member) gets past the permission gate and +-- fails on the pid lookup (masked by start_matchsubs above). +-- --------------------------------------------------------------------- +3: SET ROLE role_rg_noadmin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: must be mdb_admin to move query +3: RESET ROLE; +RESET +3: SET ROLE role_rg_admin; +SET +3: SELECT pg_resgroup_move_query(999999999, 'admin_group'); +ERROR: cannot find process: XXX +3q: ... + +-- --------------------------------------------------------------------- +-- 5. Cross-session REVOKE takes effect on the granted session's +-- next statement (the privilege is re-checked per command, not +-- cached at SET ROLE time). +-- --------------------------------------------------------------------- +4: SET ROLE role_rg_admin; +SET +4: CREATE RESOURCE GROUP rg_perm_revoke1 WITH (concurrency=1, cpu_max_percent=5); +CREATE +5: REVOKE mdb_admin FROM role_rg_admin; +REVOKE +4: CREATE RESOURCE GROUP rg_perm_revoke2 WITH (concurrency=1, cpu_max_percent=5); +ERROR: must be mdb_admin to create resource groups +4: DROP RESOURCE GROUP rg_perm_revoke1; +ERROR: must be mdb_admin to drop resource groups +4q: ... +5q: ... + +-- --------------------------------------------------------------------- +-- Cleanup. Roles must be dropped before the resource group they +-- reference, otherwise DROP RESOURCE GROUP fails with +-- "resource group is used by at least one role". +-- --------------------------------------------------------------------- +RESET ROLE; +RESET +DROP ROLE role_rg_admin; +DROP +DROP ROLE role_rg_noadmin; +DROP +DROP ROLE mdb_admin; +DROP +DROP RESOURCE GROUP rg_perm_revoke1; +DROP +DROP RESOURCE GROUP rg_perm_test; +DROP diff --git a/contrib/pg_aux_catalog/isolation2/isolation2_schedule b/contrib/pg_aux_catalog/isolation2/isolation2_schedule index 73b2a8a95a5..cb32fe3e768 100644 --- a/contrib/pg_aux_catalog/isolation2/isolation2_schedule +++ b/contrib/pg_aux_catalog/isolation2/isolation2_schedule @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + test: resgroup_mdb_admin diff --git a/contrib/pg_aux_catalog/pg_aux_catalog.c b/contrib/pg_aux_catalog/pg_aux_catalog.c index 91685561cca..958bd2f55be 100644 --- a/contrib/pg_aux_catalog/pg_aux_catalog.c +++ b/contrib/pg_aux_catalog/pg_aux_catalog.c @@ -5,6 +5,23 @@ * * contrib/pg_aux_catalog/pg_aux_catalog.c * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * *------------------------------------------------------------------------- */ #include "postgres.h" diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index af9862530d6..c62855d3089 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -35,10 +35,6 @@ else diskquota endif -ifeq "$(with_diskquota)" "yes" - recurse_targets += diskquota -endif - ifeq "$(with_gp_stats_collector)" "yes" recurse_targets += gp_stats_collector endif diff --git a/pom.xml b/pom.xml index 6eaa095fa37..0e000093399 100644 --- a/pom.xml +++ b/pom.xml @@ -352,9 +352,6 @@ code or new licensing patterns. contrib/indexscan/indexscan.c contrib/indexscan/indexscan.sql.in - contrib/pg_aux_catalog/pg_aux_catalog.c - contrib/pg_aux_catalog/isolation2/isolation2_schedule - contrib/file_fdw/init_file contrib/file_fdw/data/** diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index eaa2e099876..95afd25d0b0 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -129,6 +129,7 @@ #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" #include "catalog/oid_dispatch.h" +#include "access/transam.h" #include "cdb/cdbvars.h" #include "executor/execdesc.h" #include "lib/rbtree.h" From 60a85c36509c87cd9443485c3b6c55ab3d7fde9c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 12:37:48 +0000 Subject: [PATCH 017/167] Fix Yezzey CI shell for pipefail steps --- .github/workflows/yezzey-ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index 9913a6196c7..82511c4f701 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -127,6 +127,7 @@ jobs: echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - name: Install MinIO Client (mc) + shell: bash run: | set -exo pipefail # Download mc for Linux (amd64) @@ -135,6 +136,7 @@ jobs: sudo mv mc /usr/local/bin/mc # Make mc available system-wide - name: Configure MinIO service + shell: bash run: | set -exo pipefail # Add the MinIO service as an "alias" in mc (name it "minio-ci") @@ -331,4 +333,3 @@ jobs: echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "Cloudberry Result: ${{ needs.test-cloudberry.result }}" - From 0612130ff56c6d036b95aa3405828984242827fe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 19 Jun 2026 14:26:11 +0000 Subject: [PATCH 018/167] Fix failing ic-deb-good-opt-on CI tests: privileges and misc expected outputs - privileges.out: update expected output for terminate_nothrow test to show 4 background processes (autovacuum launcher, dtx recovery process, logical replication launcher, login monitor) instead of 0 rows, matching Cloudberry's actual pg_terminate_backend behavior - output/misc.source: change expected value from 't' to 'f' for mdb_locale_enabled() since CI builds without --with-mdblocales (ENABLE_MDBLOCALES defaults to false in configure-cloudberry.sh) --- src/test/regress/expected/privileges.out | 10 +++++++--- src/test/regress/output/misc.source | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index ee9f8fa1530..de323f54114 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -2007,9 +2007,13 @@ END$$; ALTER FUNCTION terminate_nothrow OWNER TO pg_signal_backend; SELECT backend_type FROM pg_stat_activity WHERE CASE WHEN COALESCE(usesysid, 10) = 10 THEN terminate_nothrow(pid) END; - backend_type --------------- -(0 rows) + backend_type +------------------------------ + autovacuum launcher + dtx recovery process + logical replication launcher + login monitor +(4 rows) ROLLBACK; -- test default ACLs diff --git a/src/test/regress/output/misc.source b/src/test/regress/output/misc.source index a0c63418446..f2f7c0dee32 100644 --- a/src/test/regress/output/misc.source +++ b/src/test/regress/output/misc.source @@ -613,6 +613,6 @@ CONTEXT: SQL function "equipment" during startup SELECT mdb_locale_enabled(); mdb_locale_enabled -------------------- - t + f (1 row) From 66b22288175e2b62abf2822769d7b7292cd50628 Mon Sep 17 00:00:00 2001 From: Sagittarius <101273427+MutableFire@users.noreply.github.com> Date: Tue, 23 Jun 2026 12:59:33 +0300 Subject: [PATCH 019/167] Add views for getting cumulative IC stats (#1822) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `gp_interconnect_stats` — aggregated statistics across all segments; - `gp_interconnect_stats_per_segment` — statistics grouped by segment; - `gp_interconnect_stats_per_segment_per_host` — statistics grouped by host and segment. Based on the implementation from OpenGPDB (https://github.com/open-gpdb/gpdb/pull/109). --- .github/workflows/build-cloudberry.yml | 1 + contrib/interconnect/Makefile | 6 + contrib/interconnect/README.md | 32 +++++ .../interconnect/expected/interconnect.out | 82 ++++++++++++ contrib/interconnect/ic_modules.c | 26 ++++ contrib/interconnect/ic_modules.h | 1 + contrib/interconnect/interconnect--1.0.sql | 110 ++++++++++++++++ contrib/interconnect/interconnect.control | 4 + contrib/interconnect/sql/interconnect.sql | 87 +++++++++++++ contrib/interconnect/udp/ic_udpifc.c | 120 ++++++++++++++++++ contrib/interconnect/udp/ic_udpifc.h | 28 ++++ 11 files changed, 497 insertions(+) create mode 100644 contrib/interconnect/expected/interconnect.out create mode 100644 contrib/interconnect/interconnect--1.0.sql create mode 100644 contrib/interconnect/interconnect.control create mode 100644 contrib/interconnect/sql/interconnect.sql diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 99bc67c99b7..47fb23dde27 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -298,6 +298,7 @@ jobs: "contrib/formatter_fixedwidth:installcheck", "contrib/hstore:installcheck", "contrib/indexscan:installcheck", + "contrib/interconnect:installcheck", "contrib/pg_trgm:installcheck", "contrib/indexscan:installcheck", "contrib/pgcrypto:installcheck", diff --git a/contrib/interconnect/Makefile b/contrib/interconnect/Makefile index 31489fa9148..5bb0cd188e4 100644 --- a/contrib/interconnect/Makefile +++ b/contrib/interconnect/Makefile @@ -7,6 +7,10 @@ include $(top_builddir)/contrib/interconnect/Makefile.interconnect MODULE_big = interconnect PGFILEDESC = "interconnect - inter connection module" +EXTENSION = interconnect +EXTENSION_VERSION = 1.0 +DATA = interconnect--$(EXTENSION_VERSION).sql + OBJS = \ $(WIN32RES) \ ic_common.o \ @@ -33,6 +37,8 @@ OBJS += proxy/ic_proxy_iobuf.o SHLIB_LINK += $(filter -luv, $(LIBS)) endif # enable_ic_proxy +REGRESS = interconnect + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/contrib/interconnect/README.md b/contrib/interconnect/README.md index fd9615d89c9..26a220908b3 100644 --- a/contrib/interconnect/README.md +++ b/contrib/interconnect/README.md @@ -271,3 +271,35 @@ udpifc result: Notice that: Lower TPS does not mean the protocol is slower, might means that the cpu time taken by the protocol is low. For the udpifc, it satisfies the highest tps required by `cbdb`. at the same time it occupies a lower cpu than other types of interconnect. +# interconnect statistics + +This extension provides cumulative interconnect statistics for Apache Cloudberry, including queue sizes, buffer usage, retransmits, packet errors, and other UDPIFC‑related metrics. + +It exposes three views with statistics at different aggregation levels: +- `gp_interconnect_stats` — total cluster‑wide stats; +- `gp_interconnect_stats_per_segment` — stats per segment; +- `gp_interconnect_stats_per_host` — stats grouped by host. + +## How to create the extension + +Add interconnect to shared_preload_libraries and restart the cluster. + +``` +gpconfig -c shared_preload_libraries -v \ + "$(psql -At -c \ + "SELECT array_to_string( \ + array_append( \ + string_to_array( \ + current_setting('shared_preload_libraries'), \ + ','), \ + 'interconnect'), \ + ',')" \ + postgres)" +gpstop -ra +``` + +Create the extension in your database. + +``` +CREATE EXTENSION interconnect; +``` diff --git a/contrib/interconnect/expected/interconnect.out b/contrib/interconnect/expected/interconnect.out new file mode 100644 index 00000000000..924a532e180 --- /dev/null +++ b/contrib/interconnect/expected/interconnect.out @@ -0,0 +1,82 @@ +-- Capture current interconnect stats as baseline for future comparisons +SELECT * FROM gp_interconnect_stats \gset prev_ +-- Verify that all baseline interconnect statistics are >= 0 (no negative values) +SELECT + :prev_total_recv_queue_size >= 0, + :prev_recv_queue_conting_time >= 0, + :prev_total_capacity >= 0, + :prev_capacity_counting_time >= 0, + :prev_total_buffers >= 0, + :prev_buffer_counting_time >= 0, + :prev_retransmits >= 0, + :prev_startup_cached_pkts >= 0, + :prev_mismatches >= 0, + :prev_crs_errors >= 0, + :prev_snd_pkt_num >= 0, + :prev_recv_pkt_num >= 0, + :prev_disordered_pkt_num >= 0, + :prev_duplicate_pkt_num >= 0, + :prev_recv_ack_num >= 0, + :prev_status_query_msg_num >= 0; + ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? +----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+---------- + t | t | t | t | t | t | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- Create test table to generate interconnect traffic +CREATE TABLE test_ic_data +AS SELECT generate_series(1, 1000) AS id +DISTRIBUTED RANDOMLY; +-- Re-capture current state: overwrite prev with latest values +SELECT * FROM gp_interconnect_stats \gset prev2_ +-- Check if current statistics are >= baseline values after first data insertion +SELECT + :prev2_total_recv_queue_size >= :prev_total_recv_queue_size, + :prev2_recv_queue_conting_time >= :prev_recv_queue_conting_time, + :prev2_total_capacity >= :prev_total_capacity, + :prev2_capacity_counting_time >= :prev_capacity_counting_time, + :prev2_total_buffers >= :prev_total_buffers, + :prev2_buffer_counting_time >= :prev_buffer_counting_time, + :prev2_retransmits >= :prev_retransmits, + :prev2_startup_cached_pkts >= :prev_startup_cached_pkts, + :prev2_mismatches >= :prev_mismatches, + :prev2_crs_errors >= :prev_crs_errors, + :prev2_snd_pkt_num >= :prev_snd_pkt_num, + :prev2_recv_pkt_num >= :prev_recv_pkt_num, + :prev2_disordered_pkt_num >= :prev_disordered_pkt_num, + :prev2_duplicate_pkt_num >= :prev_duplicate_pkt_num, + :prev2_recv_ack_num >= :prev_recv_ack_num, + :prev2_status_query_msg_num >= :prev_status_query_msg_num; + ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? +----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+---------- + t | t | t | t | t | t | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- Insert additional data to further test interconnect statistics changes under load +INSERT INTO test_ic_data SELECT generate_series(1001, 2000); +-- Re‑check if current statistics remain >= baseline after second data insertion +SELECT + total_recv_queue_size >= :prev2_total_recv_queue_size, + recv_queue_conting_time >= :prev2_recv_queue_conting_time, + total_capacity >= :prev2_total_capacity, + capacity_counting_time >= :prev2_capacity_counting_time, + total_buffers >= :prev2_total_buffers, + buffer_counting_time >= :prev2_buffer_counting_time, + retransmits >= :prev2_retransmits, + startup_cached_pkts >= :prev2_startup_cached_pkts, + mismatches >= :prev2_mismatches, + crs_errors >= :prev2_crs_errors, + snd_pkt_num >= :prev2_snd_pkt_num, + recv_pkt_num >= :prev2_recv_pkt_num, + disordered_pkt_num >= :prev2_disordered_pkt_num, + duplicate_pkt_num >= :prev2_duplicate_pkt_num, + recv_ack_num >= :prev2_recv_ack_num, + status_query_msg_num >= :prev2_status_query_msg_num +FROM gp_interconnect_stats; + ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? | ?column? +----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+---------- + t | t | t | t | t | t | t | t | t | t | t | t | t | t | t | t +(1 row) + +DROP TABLE test_ic_data; +DROP EXTENSION interconnect; diff --git a/contrib/interconnect/ic_modules.c b/contrib/interconnect/ic_modules.c index b582e8bdbe0..26382882ebf 100644 --- a/contrib/interconnect/ic_modules.c +++ b/contrib/interconnect/ic_modules.c @@ -16,6 +16,7 @@ #include "ic_common.h" #include "tcp/ic_tcp.h" #include "udp/ic_udpifc.h" +#include "storage/ipc.h" #ifdef ENABLE_IC_PROXY #include "proxy/ic_proxy_server.h" @@ -23,6 +24,8 @@ PG_MODULE_MAGIC; +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; + MotionIPCLayer tcp_ipc_layer = { .ic_type = INTERCONNECT_TYPE_TCP, .type_name = "tcp", @@ -141,6 +144,15 @@ MotionIPCLayer udpifc_ipc_layer = { .GetMotionSentRecordTypmod = GetMotionSentRecordTypmod, }; +static void +InterconnectShmemInit(void) +{ + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + InterconnectShmemInitUDPIFC(); +} + void _PG_init(void) { @@ -153,4 +165,18 @@ _PG_init(void) RegisterIPCLayerImpl(&tcp_ipc_layer); RegisterIPCLayerImpl(&udpifc_ipc_layer); RegisterIPCLayerImpl(&proxy_ipc_layer); + + if (Gp_interconnect_type == INTERCONNECT_TYPE_UDPIFC) + { + RequestAddinShmemSpace(MAXALIGN(sizeof(ICStatisticsShmem))); + + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = InterconnectShmemInit; + } +} + +void +_PG_fini(void) +{ + shmem_startup_hook = prev_shmem_startup_hook; } diff --git a/contrib/interconnect/ic_modules.h b/contrib/interconnect/ic_modules.h index a381d279fdd..e73b1998f3a 100644 --- a/contrib/interconnect/ic_modules.h +++ b/contrib/interconnect/ic_modules.h @@ -18,5 +18,6 @@ extern MotionIPCLayer proxy_ipc_layer; extern MotionIPCLayer udpifc_ipc_layer; extern void _PG_init(void); +extern void _PG_fini(void); #endif // INTER_CONNECT_H diff --git a/contrib/interconnect/interconnect--1.0.sql b/contrib/interconnect/interconnect--1.0.sql new file mode 100644 index 00000000000..f4aa635583e --- /dev/null +++ b/contrib/interconnect/interconnect--1.0.sql @@ -0,0 +1,110 @@ +/* contrib/interconnect/interconnect--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION interconnect" to load this file. \quit + +CREATE FUNCTION __gp_interconnect_get_stats_f_on_master( + OUT gp_segment_id smallint, + OUT total_recv_queue_size bigint, + OUT recv_queue_conting_time bigint, + OUT total_capacity bigint, + OUT capacity_counting_time bigint, + OUT total_buffers bigint, + OUT buffer_counting_time bigint, + OUT retransmits bigint, + OUT startup_cached_pkts bigint, + OUT mismatches bigint, + OUT crs_errors bigint, + OUT snd_pkt_num bigint, + OUT recv_pkt_num bigint, + OUT disordered_pkt_num bigint, + OUT duplicate_pkt_num bigint, + OUT recv_ack_num bigint, + OUT status_query_msg_num bigint +) +RETURNS SETOF record +LANGUAGE C VOLATILE EXECUTE ON MASTER +AS '$libdir/interconnect', 'gp_interconnect_get_stats'; + +CREATE FUNCTION __gp_interconnect_get_stats_f_on_segments( + OUT gp_segment_id smallint, + OUT total_recv_queue_size bigint, + OUT recv_queue_conting_time bigint, + OUT total_capacity bigint, + OUT capacity_counting_time bigint, + OUT total_buffers bigint, + OUT buffer_counting_time bigint, + OUT retransmits bigint, + OUT startup_cached_pkts bigint, + OUT mismatches bigint, + OUT crs_errors bigint, + OUT snd_pkt_num bigint, + OUT recv_pkt_num bigint, + OUT disordered_pkt_num bigint, + OUT duplicate_pkt_num bigint, + OUT recv_ack_num bigint, + OUT status_query_msg_num bigint +) +RETURNS SETOF record LANGUAGE C VOLATILE EXECUTE ON ALL SEGMENTS +AS '$libdir/interconnect', 'gp_interconnect_get_stats'; + + +-- Cummulative interconnect statistics per segment +CREATE VIEW gp_interconnect_stats_per_segment AS + SELECT c.hostname, s.* FROM ( + SELECT * FROM __gp_interconnect_get_stats_f_on_master() + UNION ALL + SELECT * FROM __gp_interconnect_get_stats_f_on_segments() + ) s + JOIN pg_catalog.gp_segment_configuration AS c + ON s.gp_segment_id = c.content AND c.role = 'p'; + +GRANT SELECT ON gp_interconnect_stats_per_segment TO public; + +-- Cummulative interconnect statistics +CREATE VIEW gp_interconnect_stats AS + SELECT + sum(total_recv_queue_size) as total_recv_queue_size + , sum(recv_queue_conting_time) as recv_queue_conting_time + , sum(total_capacity) as total_capacity + , sum(capacity_counting_time) as capacity_counting_time + , sum(total_buffers) as total_buffers + , sum(buffer_counting_time) as buffer_counting_time + , sum(retransmits) as retransmits + , sum(startup_cached_pkts) as startup_cached_pkts + , sum(mismatches) as mismatches + , sum(crs_errors) as crs_errors + , sum(snd_pkt_num) as snd_pkt_num + , sum(recv_pkt_num) as recv_pkt_num + , sum(disordered_pkt_num) as disordered_pkt_num + , sum(duplicate_pkt_num) as duplicate_pkt_num + , sum(recv_ack_num) as recv_ack_num + , sum(status_query_msg_num) as status_query_msg_num + FROM gp_interconnect_stats_per_segment; + +GRANT SELECT ON gp_interconnect_stats TO public; + +-- Cummulative interconnect statistics grouped by host +CREATE VIEW gp_interconnect_stats_per_host AS + SELECT + hostname + , sum(total_recv_queue_size) as total_recv_queue_size + , sum(recv_queue_conting_time) as recv_queue_conting_time + , sum(total_capacity) as total_capacity + , sum(capacity_counting_time) as capacity_counting_time + , sum(total_buffers) as total_buffers + , sum(buffer_counting_time) as buffer_counting_time + , sum(retransmits) as retransmits + , sum(startup_cached_pkts) as startup_cached_pkts + , sum(mismatches) as mismatches + , sum(crs_errors) as crs_errors + , sum(snd_pkt_num) as snd_pkt_num + , sum(recv_pkt_num) as recv_pkt_num + , sum(disordered_pkt_num) as disordered_pkt_num + , sum(duplicate_pkt_num) as duplicate_pkt_num + , sum(recv_ack_num) as recv_ack_num + , sum(status_query_msg_num) as status_query_msg_num + FROM gp_interconnect_stats_per_segment + GROUP BY hostname; + +GRANT SELECT ON gp_interconnect_stats_per_host TO public; diff --git a/contrib/interconnect/interconnect.control b/contrib/interconnect/interconnect.control new file mode 100644 index 00000000000..0cf63e411c9 --- /dev/null +++ b/contrib/interconnect/interconnect.control @@ -0,0 +1,4 @@ +comment = 'Cummulative statistics from UDPIFC interconnect protocol' +default_version = '1.0' +relocatable = false +schema = public diff --git a/contrib/interconnect/sql/interconnect.sql b/contrib/interconnect/sql/interconnect.sql new file mode 100644 index 00000000000..4e6555b6b82 --- /dev/null +++ b/contrib/interconnect/sql/interconnect.sql @@ -0,0 +1,87 @@ +-- start_ignore +\! gpconfig -c shared_preload_libraries -v "interconnect" +\! gpstop -raiq +\c +DROP TABLE IF EXISTS test_ic_data; +CREATE EXTENSION IF NOT EXISTS interconnect; +-- end_ignore + +-- Capture current interconnect stats as baseline for future comparisons +SELECT * FROM gp_interconnect_stats \gset prev_ + +-- Verify that all baseline interconnect statistics are >= 0 (no negative values) +SELECT + :prev_total_recv_queue_size >= 0, + :prev_recv_queue_conting_time >= 0, + :prev_total_capacity >= 0, + :prev_capacity_counting_time >= 0, + :prev_total_buffers >= 0, + :prev_buffer_counting_time >= 0, + :prev_retransmits >= 0, + :prev_startup_cached_pkts >= 0, + :prev_mismatches >= 0, + :prev_crs_errors >= 0, + :prev_snd_pkt_num >= 0, + :prev_recv_pkt_num >= 0, + :prev_disordered_pkt_num >= 0, + :prev_duplicate_pkt_num >= 0, + :prev_recv_ack_num >= 0, + :prev_status_query_msg_num >= 0; + +-- Create test table to generate interconnect traffic +CREATE TABLE test_ic_data +AS SELECT generate_series(1, 1000) AS id +DISTRIBUTED RANDOMLY; + +-- Re-capture current state: overwrite prev with latest values +SELECT * FROM gp_interconnect_stats \gset prev2_ + +-- Check if current statistics are >= baseline values after first data insertion +SELECT + :prev2_total_recv_queue_size >= :prev_total_recv_queue_size, + :prev2_recv_queue_conting_time >= :prev_recv_queue_conting_time, + :prev2_total_capacity >= :prev_total_capacity, + :prev2_capacity_counting_time >= :prev_capacity_counting_time, + :prev2_total_buffers >= :prev_total_buffers, + :prev2_buffer_counting_time >= :prev_buffer_counting_time, + :prev2_retransmits >= :prev_retransmits, + :prev2_startup_cached_pkts >= :prev_startup_cached_pkts, + :prev2_mismatches >= :prev_mismatches, + :prev2_crs_errors >= :prev_crs_errors, + :prev2_snd_pkt_num >= :prev_snd_pkt_num, + :prev2_recv_pkt_num >= :prev_recv_pkt_num, + :prev2_disordered_pkt_num >= :prev_disordered_pkt_num, + :prev2_duplicate_pkt_num >= :prev_duplicate_pkt_num, + :prev2_recv_ack_num >= :prev_recv_ack_num, + :prev2_status_query_msg_num >= :prev_status_query_msg_num; + +-- Insert additional data to further test interconnect statistics changes under load +INSERT INTO test_ic_data SELECT generate_series(1001, 2000); + +-- Re‑check if current statistics remain >= baseline after second data insertion +SELECT + total_recv_queue_size >= :prev2_total_recv_queue_size, + recv_queue_conting_time >= :prev2_recv_queue_conting_time, + total_capacity >= :prev2_total_capacity, + capacity_counting_time >= :prev2_capacity_counting_time, + total_buffers >= :prev2_total_buffers, + buffer_counting_time >= :prev2_buffer_counting_time, + retransmits >= :prev2_retransmits, + startup_cached_pkts >= :prev2_startup_cached_pkts, + mismatches >= :prev2_mismatches, + crs_errors >= :prev2_crs_errors, + snd_pkt_num >= :prev2_snd_pkt_num, + recv_pkt_num >= :prev2_recv_pkt_num, + disordered_pkt_num >= :prev2_disordered_pkt_num, + duplicate_pkt_num >= :prev2_duplicate_pkt_num, + recv_ack_num >= :prev2_recv_ack_num, + status_query_msg_num >= :prev2_status_query_msg_num +FROM gp_interconnect_stats; + +DROP TABLE test_ic_data; +DROP EXTENSION interconnect; + +-- start_ignore +\! gpconfig -r shared_preload_libraries +\! gpstop -raiq +-- end_ignore diff --git a/contrib/interconnect/udp/ic_udpifc.c b/contrib/interconnect/udp/ic_udpifc.c index d11e4577cd6..23de982d89b 100644 --- a/contrib/interconnect/udp/ic_udpifc.c +++ b/contrib/interconnect/udp/ic_udpifc.c @@ -41,6 +41,7 @@ #include "access/transam.h" #include "access/xact.h" #include "common/ip.h" +#include "funcapi.h" #include "nodes/execnodes.h" #include "nodes/pg_list.h" #include "nodes/print.h" @@ -51,6 +52,8 @@ #include "pgstat.h" #include "postmaster/postmaster.h" #include "storage/latch.h" +#include "storage/lock.h" +#include "storage/pg_shmem.h" #include "storage/pmsignal.h" #include "utils/builtins.h" #include "utils/guc.h" @@ -714,6 +717,8 @@ typedef struct ICStatistics /* Statistics for UDP interconnect. */ static ICStatistics ic_statistics; +static ICStatisticsShmem *pICStatisticsShmem = NULL; + /* UDP listen fd */ int UDP_listenerFd; @@ -1814,6 +1819,27 @@ ic_reset_pthread_sigmasks(sigset_t *sigs) return; } +void +InterconnectShmemInitUDPIFC(void) +{ + bool found; + pICStatisticsShmem = ShmemInitStruct("global interconnect statistics", + sizeof(ICStatisticsShmem), &found); + if (pICStatisticsShmem == NULL) + { + ereport(FATAL, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("not enough shared memory for global interconnect statistics"))); + } + + if (!found) + memset(pICStatisticsShmem, 0, sizeof(*pICStatisticsShmem)); + + int tranche_id = LWLockNewTrancheId(); + LWLockRegisterTranche(tranche_id, "IC Statistics"); + LWLockInitialize(&pICStatisticsShmem->lock, tranche_id); +} + /* * InitMotionUDPIFC * Initialize UDP specific comms, and create rx-thread. @@ -3901,6 +3927,30 @@ chunkTransportStateEntryInitialized(ChunkTransportState *transportStates, return pEntry->valid; } +/* Append local interconnect stats to global cummulative stats. */ +static void +updateGlobalInterconnectStats(void) +{ + LWLockAcquire(&pICStatisticsShmem->lock, LW_EXCLUSIVE); + pICStatisticsShmem->totalRecvQueueSize += ic_statistics.totalRecvQueueSize; + pICStatisticsShmem->recvQueueSizeCountingTime += ic_statistics.recvQueueSizeCountingTime; + pICStatisticsShmem->totalCapacity += ic_statistics.totalCapacity; + pICStatisticsShmem->capacityCountingTime += ic_statistics.capacityCountingTime; + pICStatisticsShmem->totalBuffers += ic_statistics.totalBuffers; + pICStatisticsShmem->bufferCountingTime += ic_statistics.bufferCountingTime; + pICStatisticsShmem->retransmits += ic_statistics.retransmits; + pICStatisticsShmem->startupCachedPktNum += ic_statistics.startupCachedPktNum; + pICStatisticsShmem->mismatchNum += ic_statistics.mismatchNum; + pICStatisticsShmem->crcErrors += ic_statistics.crcErrors; + pICStatisticsShmem->sndPktNum += ic_statistics.sndPktNum; + pICStatisticsShmem->recvPktNum += ic_statistics.recvPktNum; + pICStatisticsShmem->disorderedPktNum += ic_statistics.disorderedPktNum; + pICStatisticsShmem->duplicatedPktNum += ic_statistics.duplicatedPktNum; + pICStatisticsShmem->recvAckNum += ic_statistics.recvAckNum; + pICStatisticsShmem->statusQueryMsgNum += ic_statistics.statusQueryMsgNum; + LWLockRelease(&pICStatisticsShmem->lock); +} + /* * computeNetworkStatistics * Compute the max/min/avg network statistics. @@ -4207,6 +4257,7 @@ TeardownUDPIFCInterconnect_Internal(ChunkTransportState *transportStates, (minRtt == ~((uint64) 0) ? 0 : minRtt), (minDev == ~((uint64) 0) ? 0 : minDev), avgRtt, avgDev, maxRtt, maxDev, snd_control_info.cwnd, ic_statistics.statusQueryMsgNum); + updateGlobalInterconnectStats(); ic_control_info.isSender = false; memset(&ic_statistics, 0, sizeof(ICStatistics)); @@ -8224,3 +8275,72 @@ MlPutRxBufferIFC(ChunkTransportState *transportStates, int motNodeID, int route) if (param.msg.len != 0) sendAckWithParam(¶m); } + +PG_FUNCTION_INFO_V1(gp_interconnect_get_stats); + +Datum +gp_interconnect_get_stats(PG_FUNCTION_ARGS) +{ + if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) + { + ereport(WARNING, + (errcode(ERRCODE_WARNING_GP_INTERCONNECTION), + errmsg("Interconnect statistics are collected only for UDPIFC protocol"))); + PG_RETURN_NULL(); + } + + /* + * Build a tuple descriptor for our result type + * The number and type of attributes have to match the definition of the + * view gp_interconnect_stats_per_segment + */ + enum {NUM_IC_STATS_ELEM = 17}; + TupleDesc tupdesc = CreateTemplateTupleDesc(NUM_IC_STATS_ELEM); + + TupleDescInitEntry(tupdesc, 1, "segid", INT2OID, -1, 0); + TupleDescInitEntry(tupdesc, 2, "total_recv_queue_size", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 3, "recv_queue_conting_time", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 4, "total_capacity", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 5, "capacity_counting_time", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 6, "total_buffers", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 7, "buffer_counting_time", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 8, "retransmits", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 9, "startup_cached_pkts", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 10, "mismatches", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 11, "crs_errors", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 12, "snd_pkt_num", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 13, "recv_pkt_num", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 14, "disordered_pkt_num", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 15, "duplicate_pkt_num", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 16, "recv_ack_num", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, 17, "status_query_msg_num", INT8OID, -1, 0); + tupdesc = BlessTupleDesc(tupdesc); + + Datum values[NUM_IC_STATS_ELEM]; + bool nulls[NUM_IC_STATS_ELEM] = {0}; + + LWLockAcquire(&pICStatisticsShmem->lock, LW_SHARED); + values[0] = Int32GetDatum(GpIdentity.segindex); + values[1] = Int64GetDatum(pICStatisticsShmem->totalRecvQueueSize); + values[2] = Int64GetDatum(pICStatisticsShmem->recvQueueSizeCountingTime); + values[3] = Int64GetDatum(pICStatisticsShmem->totalCapacity); + values[4] = Int64GetDatum(pICStatisticsShmem->capacityCountingTime); + values[5] = Int64GetDatum(pICStatisticsShmem->totalBuffers); + values[6] = Int64GetDatum(pICStatisticsShmem->bufferCountingTime); + values[7] = Int64GetDatum(pICStatisticsShmem->retransmits); + values[8] = Int64GetDatum(pICStatisticsShmem->startupCachedPktNum); + values[9] = Int64GetDatum(pICStatisticsShmem->mismatchNum); + values[10] = Int64GetDatum(pICStatisticsShmem->crcErrors); + values[11] = Int64GetDatum(pICStatisticsShmem->sndPktNum); + values[12] = Int64GetDatum(pICStatisticsShmem->recvPktNum); + values[13] = Int64GetDatum(pICStatisticsShmem->disorderedPktNum); + values[14] = Int64GetDatum(pICStatisticsShmem->duplicatedPktNum); + values[15] = Int64GetDatum(pICStatisticsShmem->recvAckNum); + values[16] = Int64GetDatum(pICStatisticsShmem->statusQueryMsgNum); + LWLockRelease(&pICStatisticsShmem->lock); + + HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + + PG_RETURN_DATUM(result); +} diff --git a/contrib/interconnect/udp/ic_udpifc.h b/contrib/interconnect/udp/ic_udpifc.h index af3ca72ba3b..346f6ea85cb 100644 --- a/contrib/interconnect/udp/ic_udpifc.h +++ b/contrib/interconnect/udp/ic_udpifc.h @@ -17,6 +17,7 @@ #include "nodes/execnodes.h" /* ExecSlice, SliceTable */ #include "miscadmin.h" #include "libpq/libpq-be.h" +#include "storage/lwlock.h" #include "utils/builtins.h" #include "utils/memutils.h" @@ -212,4 +213,31 @@ extern void dumpICBufferList(ICBufferList * list, const char *fname); extern void dumpUnackQueueRing(const char *fname); extern void dumpConnections(ChunkTransportStateEntry * pEntry, const char *fname); +/* + * Keeps various statistics about interconnect internal. + * Also those numbers are expected to grow big, hence uint64. + */ +typedef struct ICStatisticsShmem +{ + LWLock lock; /* mutex for synchronizing access to statistics data */ + uint64 totalRecvQueueSize; /* receive queue size sum when main thread is trying to get a packet */ + uint64 recvQueueSizeCountingTime; /* counting times when computing totalRecvQueueSize */ + uint64 totalCapacity; /* the capacity sum when packets are tried to be sent */ + uint64 capacityCountingTime; /* counting times used to compute totalCapacity */ + uint64 totalBuffers; /* total buffers available when sending packets */ + uint64 bufferCountingTime; /* counting times when compute totalBuffers */ + uint64 retransmits; /* the number of packet retransmits */ + uint64 startupCachedPktNum; /* number of packets cached during connection startup */ + uint64 mismatchNum; /* the number of mismatched packets received */ + uint64 crcErrors; /* the number of crc errors */ + uint64 sndPktNum; /* the number of packets sent by sender */ + uint64 recvPktNum; /* the number of packets received by receiver */ + uint64 disorderedPktNum; /* disordered packet number */ + uint64 duplicatedPktNum; /* duplicate packet number */ + uint64 recvAckNum; /* the number of Acks received */ + uint64 statusQueryMsgNum; /* the number of status query messages sent */ +} ICStatisticsShmem; + +void InterconnectShmemInitUDPIFC(void); + #endif // IC_UDP_INTERFACE_H From f68292bddffb7c69ba2f6e78fde0e888038199bb Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 29 Jul 2025 17:21:42 +0800 Subject: [PATCH 020/167] License: add & clean up license headers and files This commit performs a comprehensive license compliance cleanup to align with release requirements, which are pointed out by Incubator PMC review. The main changes include: 1. Add License Headers: Added the standard Apache License Version 2.0 header to numerous source files that were missing it. This covers multiple file types, including YAML, Markdown, SQL, C/C++, Python, and shell scripts. These files are originally created by the cloudberry community. 2. Simplify LICENSE and NOTICE: - Restructured the root LICENSE file for better clarity. - Cleaned up the NOTICE file by removing redundant information which have been listed in the LICENSE. 3. Remove the unused deployment docs from the `deploy/build`, which can help us manage the file licenses. 4. Update RAT Configuration: Updated `pom.xml` to reflect the changes of the file license headers and attribution. See: https://github.com/apache/cloudberry/issues/1236 --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0e000093399..1bd58894929 100644 --- a/pom.xml +++ b/pom.xml @@ -1243,6 +1243,7 @@ code or new licensing patterns. aclocal.m4 python-dependencies.txt + The frontend should also be prepared to handle an ErrorMessage - response to SSLRequest from the server. This would only occur if - the server predates the addition of SSL support - to PostgreSQL. (Such servers are now very ancient, - and likely do not exist in the wild anymore.) + response to SSLRequest from the server. The frontend should not display + this error message to the user/application, since the server has not been + authenticated + (CVE-2024-10977). In this case the connection must be closed, but the frontend might choose to open a fresh connection and proceed without requesting SSL. @@ -1604,12 +1604,13 @@ SELCT 1/0; The frontend should also be prepared to handle an ErrorMessage - response to GSSENCRequest from the server. This would only occur if - the server predates the addition of GSSAPI encryption - support to PostgreSQL. In this case the - connection must be closed, but the frontend might choose to open a fresh - connection and proceed without requesting GSSAPI - encryption. + response to GSSENCRequest from the server. The frontend should not display + this error message to the user/application, since the server has not been + authenticated + (CVE-2024-10977). + In this case the connection must be closed, but the frontend might choose + to open a fresh connection and proceed without requesting + GSSAPI encryption. diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 46e8540004e..0d4d2fed864 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -3168,16 +3168,13 @@ PQconnectPoll(PGconn *conn) { /* * Server failure of some sort, such as failure to - * fork a backend process. We need to process and - * report the error message, which might be formatted - * according to either protocol 2 or protocol 3. - * Rather than duplicate the code for that, we flip - * into AWAITING_RESPONSE state and let the code there - * deal with it. Note we have *not* consumed the "E" - * byte here. + * fork a backend process. Don't bother retrieving + * the error message; we should not trust it as the + * server has not been authenticated yet. */ - conn->status = CONNECTION_AWAITING_RESPONSE; - goto keep_going; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("server sent an error response during SSL exchange\n")); + goto error_return; } else { From 50638318a2389bdc62fd4850b761cd16d9be06f2 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 31 Mar 2023 11:01:51 +1300 Subject: [PATCH 049/167] Parallel Hash Full Join. Full and right outer joins were not supported in the initial implementation of Parallel Hash Join because of deadlock hazards (see discussion). Therefore FULL JOIN inhibited parallelism, as the other join strategies can't do that in parallel either. Add a new PHJ phase PHJ_BATCH_SCAN that scans for unmatched tuples on the inner side of one batch's hash table. For now, sidestep the deadlock problem by terminating parallelism there. The last process to arrive at that phase emits the unmatched tuples, while others detach and are free to go and work on other batches, if there are any, but otherwise they finish the join early. That unfairness is considered acceptable for now, because it's better than no parallelism at all. The build and probe phases are run in parallel, and the new scan-for-unmatched phase, while serial, is usually applied to the smaller of the two relations and is either limited by some multiple of work_mem, or it's too big and is partitioned into batches and then the situation is improved by batch-level parallelism. Author: Melanie Plageman Author: Thomas Munro Reviewed-by: Thomas Munro Discussion: https://postgr.es/m/CA%2BhUKG%2BA6ftXPz4oe92%2Bx8Er%2BxpGZqto70-Q_ERwRaSyA%3DafNg%40mail.gmail.com --- src/backend/executor/nodeHash.c | 179 +++++++++++++++++++++++- src/backend/executor/nodeHashjoin.c | 95 ++++++++----- src/backend/optimizer/path/joinpath.c | 14 +- src/include/executor/hashjoin.h | 6 +- src/include/executor/nodeHash.h | 3 + src/test/regress/expected/join_hash.out | 65 ++++++++- src/test/regress/sql/join_hash.sql | 27 +++- 7 files changed, 333 insertions(+), 56 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index a236f5b4819..e3e1d5a6d87 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2395,6 +2395,69 @@ ExecPrepHashTableForUnmatched(HashJoinState *hjstate) hjstate->hj_CurTuple = NULL; } +/* + * Decide if this process is allowed to run the unmatched scan. If so, the + * batch barrier is advanced to PHJ_BATCH_SCAN and true is returned. + * Otherwise the batch is detached and false is returned. + */ +bool +ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate) +{ + HashJoinTable hashtable = hjstate->hj_HashTable; + int curbatch = hashtable->curbatch; + ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; + + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE); + + /* + * It would not be deadlock-free to wait on the batch barrier, because it + * is in PHJ_BATCH_PROBE phase, and thus processes attached to it have + * already emitted tuples. Therefore, we'll hold a wait-free election: + * only one process can continue to the next phase, and all others detach + * from this batch. They can still go any work on other batches, if there + * are any. + */ + if (!BarrierArriveAndDetachExceptLast(&batch->batch_barrier)) + { + /* This process considers the batch to be done. */ + hashtable->batches[hashtable->curbatch].done = true; + + /* Make sure any temporary files are closed. */ + sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); + sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); + + /* + * Track largest batch we've seen, which would normally happen in + * ExecHashTableDetachBatch(). + */ + hashtable->spacePeak = + Max(hashtable->spacePeak, + batch->size + sizeof(dsa_pointer_atomic) * hashtable->nbuckets); + hashtable->curbatch = -1; + return false; + } + + /* Now we are alone with this batch. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); + Assert(BarrierParticipants(&batch->batch_barrier) == 1); + + /* + * Has another process decided to give up early and command all processes + * to skip the unmatched scan? + */ + if (batch->skip_unmatched) + { + hashtable->batches[hashtable->curbatch].done = true; + ExecHashTableDetachBatch(hashtable); + return false; + } + + /* Now prepare the process local state, just as for non-parallel join. */ + ExecPrepHashTableForUnmatched(hjstate); + + return true; +} + /* * ExecScanHashTableForUnmatched * scan the hash table for unmatched inner tuples @@ -2469,6 +2532,72 @@ ExecScanHashTableForUnmatched(HashJoinState *hjstate, ExprContext *econtext) return false; } +/* + * ExecParallelScanHashTableForUnmatched + * scan the hash table for unmatched inner tuples, in parallel join + * + * On success, the inner tuple is stored into hjstate->hj_CurTuple and + * econtext->ecxt_innertuple, using hjstate->hj_HashTupleSlot as the slot + * for the latter. + */ +bool +ExecParallelScanHashTableForUnmatched(HashJoinState *hjstate, + ExprContext *econtext) +{ + HashJoinTable hashtable = hjstate->hj_HashTable; + HashJoinTuple hashTuple = hjstate->hj_CurTuple; + + for (;;) + { + /* + * hj_CurTuple is the address of the tuple last returned from the + * current bucket, or NULL if it's time to start scanning a new + * bucket. + */ + if (hashTuple != NULL) + hashTuple = ExecParallelHashNextTuple(hashtable, hashTuple); + else if (hjstate->hj_CurBucketNo < hashtable->nbuckets) + hashTuple = ExecParallelHashFirstTuple(hashtable, + hjstate->hj_CurBucketNo++); + else + break; /* finished all buckets */ + + while (hashTuple != NULL) + { + if (!HeapTupleHeaderHasMatch(HJTUPLE_MINTUPLE(hashTuple))) + { + TupleTableSlot *inntuple; + + /* insert hashtable's tuple into exec slot */ + inntuple = ExecStoreMinimalTuple(HJTUPLE_MINTUPLE(hashTuple), + hjstate->hj_HashTupleSlot, + false); /* do not pfree */ + econtext->ecxt_innertuple = inntuple; + + /* + * Reset temp memory each time; although this function doesn't + * do any qual eval, the caller will, so let's keep it + * parallel to ExecScanHashBucket. + */ + ResetExprContext(econtext); + + hjstate->hj_CurTuple = hashTuple; + return true; + } + + hashTuple = ExecParallelHashNextTuple(hashtable, hashTuple); + } + + /* allow this loop to be cancellable */ + CHECK_FOR_INTERRUPTS(); + } + + /* + * no more unmatched tuples + */ + return false; +} + /* * ExecHashTableReset * @@ -3797,6 +3926,7 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) accessor->shared = shared; accessor->preallocated = 0; accessor->done = false; + accessor->outer_eof = false; accessor->inner_tuples = sts_attach(ParallelHashJoinBatchInner(shared), hashtable->hjstate->worker_id, @@ -3842,25 +3972,62 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) { int curbatch = hashtable->curbatch; ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; + bool attached = true; /* Make sure any temporary files are closed. */ sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); - /* Detach from the batch we were last working on. */ + /* After attaching we always get at least to PHJ_BATCH_PROBE. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE || + BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); + + /* + * If we're abandoning the PHJ_BATCH_PROBE phase early without having + * reached the end of it, it means the plan doesn't want any more + * tuples, and it is happy to abandon any tuples buffered in this + * process's subplans. For correctness, we can't allow any process to + * execute the PHJ_BATCH_SCAN phase, because we will never have the + * complete set of match bits. Therefore we skip emitting unmatched + * tuples in all backends (if this is a full/right join), as if those + * tuples were all due to be emitted by this process and it has + * abandoned them too. + */ /* * CBDB_PARALLEL: Parallel Hash Left Anti Semi (Not-In) Join(parallel-aware) * If phs_lasj_has_null is true, that means we have found null when building hash table, * there were no batches to detach. */ - if (!hashtable->parallel_state->phs_lasj_has_null && BarrierArriveAndDetach(&batch->batch_barrier)) + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + !hashtable->parallel_state->phs_lasj_has_null && /* CBDB_PARALLEL */ + !hashtable->batches[curbatch].outer_eof) + { + /* + * This flag may be written to by multiple backends during + * PHJ_BATCH_PROBE phase, but will only be read in PHJ_BATCH_SCAN + * phase so requires no extra locking. + */ + batch->skip_unmatched = true; + } + + /* + * Even if we aren't doing a full/right outer join, we'll step through + * the PHJ_BATCH_SCAN phase just to maintain the invariant that + * freeing happens in PHJ_BATCH_FREE, but that'll be wait-free. + */ + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */) + attached = BarrierArriveAndDetachExceptLast(&batch->batch_barrier); + if (attached && BarrierArriveAndDetach(&batch->batch_barrier)) { /* - * Technically we shouldn't access the barrier because we're no - * longer attached, but since there is no way it's moving after - * this point it seems safe to make the following assertion. + * We are not longer attached to the batch barrier, but we're the + * process that was chosen to free resources and it's safe to + * assert the current phase. The ParallelHashJoinBatch can't go + * away underneath us while we are attached to the build barrier, + * making this access safe. */ - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_DONE); + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_FREE); /* Free shared chunks and buckets. */ while (DsaPointerIsValid(batch->chunks)) diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 88eaaa10cef..53bb29be7af 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -81,11 +81,12 @@ * aren't enough to go around. For each batch there is a separate barrier * with the following phases: * - * PHJ_BATCH_ELECTING -- initial state - * PHJ_BATCH_ALLOCATING -- one allocates buckets - * PHJ_BATCH_LOADING -- all load the hash table from disk - * PHJ_BATCH_PROBING -- all probe - * PHJ_BATCH_DONE -- end + * PHJ_BATCH_ELECT -- initial state + * PHJ_BATCH_ALLOCATE* -- one allocates buckets + * PHJ_BATCH_LOAD -- all load the hash table from disk + * PHJ_BATCH_PROBE -- all probe + * PHJ_BATCH_SCAN* -- one does full/right unmatched scan + * PHJ_BATCH_FREE* -- one frees memory * * Batch 0 is a special case, because it starts out in phase * PHJ_BATCH_PROBING; populating batch 0's hash table is done during @@ -101,10 +102,11 @@ * finished. Practically, that means that we never emit a tuple while attached * to a barrier, unless the barrier has reached a phase that means that no * process will wait on it again. We emit tuples while attached to the build - * barrier in phase PHJ_BUILD_RUNNING, and to a per-batch barrier in phase - * PHJ_BATCH_PROBING. These are advanced to PHJ_BUILD_DONE and PHJ_BATCH_DONE - * respectively without waiting, using BarrierArriveAndDetach(). The last to - * detach receives a different return value so that it knows that it's safe to + * barrier in phase PHJ_BUILD_RUN, and to a per-batch barrier in phase + * PHJ_BATCH_PROBE. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN + * respectively without waiting, using BarrierArriveAndDetach() and + * BarrierArriveAndDetachExceptLast() respectively. The last to detach + * receives a different return value so that it knows that it's safe to * clean up. Any straggler process that attaches after that phase is reached * will see that it's too late to participate or access the relevant shared * memory objects. @@ -523,8 +525,23 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) if (HJ_FILL_INNER(node)) { /* set up to scan for unmatched inner tuples */ - ExecPrepHashTableForUnmatched(node); - node->hj_JoinState = HJ_FILL_INNER_TUPLES; + if (parallel) + { + /* + * Only one process is currently allow to handle + * each batch's unmatched tuples, in a parallel + * join. + */ + if (ExecParallelPrepHashTableForUnmatched(node)) + node->hj_JoinState = HJ_FILL_INNER_TUPLES; + else + node->hj_JoinState = HJ_NEED_NEW_BATCH; + } + else + { + ExecPrepHashTableForUnmatched(node); + node->hj_JoinState = HJ_FILL_INNER_TUPLES; + } } else node->hj_JoinState = HJ_NEED_NEW_BATCH; @@ -635,25 +652,13 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) { node->hj_MatchedOuter = true; - if (parallel) - { - /* - * Full/right outer joins are currently not supported - * for parallel joins, so we don't need to set the - * match bit. Experiments show that it's worth - * avoiding the shared memory traffic on large - * systems. - */ - Assert(!HJ_FILL_INNER(node)); - } - else - { - /* - * This is really only needed if HJ_FILL_INNER(node), - * but we'll avoid the branch and just set it always. - */ + + /* + * This is really only needed if HJ_FILL_INNER(node), but + * we'll avoid the branch and just set it always. + */ + if (!HeapTupleHeaderHasMatch(HJTUPLE_MINTUPLE(node->hj_CurTuple))) HeapTupleHeaderSetMatch(HJTUPLE_MINTUPLE(node->hj_CurTuple)); - } /* In an antijoin, we never return a matched tuple */ if (node->js.jointype == JOIN_ANTI || @@ -712,7 +717,8 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) * so any unmatched inner tuples in the hashtable have to be * emitted before we continue to the next batch. */ - if (!ExecScanHashTableForUnmatched(node, econtext)) + if (!(parallel ? ExecParallelScanHashTableForUnmatched(node, econtext) + : ExecScanHashTableForUnmatched(node, econtext))) { /* no more unmatched tuples */ node->hj_JoinState = HJ_NEED_NEW_BATCH; @@ -1271,6 +1277,8 @@ ExecParallelHashJoinOuterGetTuple(PlanState *outerNode, } /* End of this batch */ + hashtable->batches[curbatch].outer_eof = true; + return NULL; } @@ -1543,15 +1551,34 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * hash table stays alive until everyone's finished * probing it, but no participant is allowed to wait at * this barrier again (or else a deadlock could occur). - * All attached participants must eventually call - * BarrierArriveAndDetach() so that the final phase - * PHJ_BATCH_DONE can be reached. + * All attached participants must eventually detach from + * the barrier and one worker must advance the phase so + * that the final phase is reached. */ ExecParallelHashTableSetCurrentBatch(hashtable, batchno); sts_begin_parallel_scan(hashtable->batches[batchno].outer_tuples); + return true; + case PHJ_BATCH_SCAN: + + /* + * In principle, we could help scan for unmatched tuples, + * since that phase is already underway (the thing we + * can't do under current deadlock-avoidance rules is wait + * for others to arrive at PHJ_BATCH_SCAN, because + * PHJ_BATCH_PROBE emits tuples, but in this case we just + * got here without waiting). That is not yet done. For + * now, we just detach and go around again. We have to + * use ExecHashTableDetachBatch() because there's a small + * chance we'll be the last to detach, and then we're + * responsible for freeing memory. + */ + ExecParallelHashTableSetCurrentBatch(hashtable, batchno); + hashtable->batches[batchno].done = true; + ExecHashTableDetachBatch(hashtable); + break; - case PHJ_BATCH_DONE: + case PHJ_BATCH_FREE: /* * Already done. Detach and go around again (if any diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index d4c2b793bb5..b5e7ef3b60c 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -2327,15 +2327,9 @@ hash_inner_and_outer(PlannerInfo *root, * able to properly guarantee uniqueness. Similarly, we can't handle * JOIN_FULL and JOIN_RIGHT, because they can produce false null * extended rows. Also, the resulting path must not be parameterized. - * We would be able to support JOIN_FULL and JOIN_RIGHT for Parallel - * Hash, since in that case we're back to a single hash table with a - * single set of match bits for each batch, but that will require - * figuring out a deadlock-free way to wait for the probe to finish. */ if (joinrel->consider_parallel && save_jointype != JOIN_UNIQUE_OUTER && - save_jointype != JOIN_FULL && - save_jointype != JOIN_RIGHT && outerrel->partial_pathlist != NIL && bms_is_empty(joinrel->lateral_relids)) { @@ -2372,9 +2366,13 @@ hash_inner_and_outer(PlannerInfo *root, * total inner path will also be parallel-safe, but if not, we'll * have to search for the cheapest safe, unparameterized inner * path. If doing JOIN_UNIQUE_INNER, we can't use any alternative - * inner path. + * inner path. If full or right join, we can't use parallelism + * (building the hash table in each backend) because no one + * process has all the match bits. */ - if (cheapest_total_inner->parallel_safe) + if (save_jointype == JOIN_FULL || save_jointype == JOIN_RIGHT) + cheapest_safe_inner = NULL; + else if (cheapest_total_inner->parallel_safe) cheapest_safe_inner = cheapest_total_inner; else if (save_jointype != JOIN_UNIQUE_INNER) cheapest_safe_inner = diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index e324e67d914..9e243c47847 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -195,6 +195,7 @@ typedef struct ParallelHashJoinBatch size_t ntuples; /* number of tuples loaded */ size_t old_ntuples; /* number of tuples before repartitioning */ bool space_exhausted; + bool skip_unmatched; /* whether to abandon unmatched scan */ /* * Variable-sized SharedTuplestore objects follow this struct in memory. @@ -239,7 +240,7 @@ typedef struct ParallelHashJoinBatchAccessor size_t estimated_size; /* size of partition on disk */ size_t old_ntuples; /* how many tuples before repartitioning? */ bool at_least_one_chunk; /* has this backend allocated a chunk? */ - + bool outer_eof; /* has this process hit end of batch? */ bool done; /* flag to remember that a batch is done */ SharedTuplestoreAccessor *inner_tuples; SharedTuplestoreAccessor *outer_tuples; @@ -306,7 +307,8 @@ typedef struct ParallelHashJoinState #define PHJ_BATCH_ALLOCATING 1 #define PHJ_BATCH_LOADING 2 #define PHJ_BATCH_PROBING 3 -#define PHJ_BATCH_DONE 4 +#define PHJ_BATCH_SCAN 4 +#define PHJ_BATCH_FREE 5 /* The phases of batch growth while hashing, for grow_batches_barrier. */ #define PHJ_GROW_BATCHES_ELECTING 0 diff --git a/src/include/executor/nodeHash.h b/src/include/executor/nodeHash.h index 993de4519b5..36549376ef9 100644 --- a/src/include/executor/nodeHash.h +++ b/src/include/executor/nodeHash.h @@ -64,9 +64,12 @@ extern bool ExecScanHashBucket(HashState *hashState, HashJoinState *hjstate, extern bool ExecParallelScanHashBucket(HashState *hashState, HashJoinState *hjstate, ExprContext *econtext); extern void ExecPrepHashTableForUnmatched(HashJoinState *hjstate); +extern bool ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate); extern bool ExecScanHashTableForUnmatched(HashJoinState *hjstate, ExprContext *econtext); extern void ExecHashTableReset(HashState *hashState, HashJoinTable hashtable); +extern bool ExecParallelScanHashTableForUnmatched(HashJoinState *hjstate, + ExprContext *econtext); extern void ExecHashTableResetMatchFlags(HashJoinTable hashtable); extern void ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew, uint64 operatorMemKB, diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 5171a7d9cf3..250704efbd7 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -315,6 +315,13 @@ $$); t | f (1 row) +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); + count +------- + 20000 +(1 row) + rollback to settings; -- The "bad" case: during execution we need to increase number of -- batches; in this case we plan for 1 batch, and increase at least a @@ -816,8 +823,9 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); @@ -841,7 +849,32 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); + QUERY PLAN +------------------------------------------------------------- + Finalize Aggregate + -> Gather + Workers Planned: 2 + -> Partial Aggregate + -> Parallel Hash Full Join + Hash Cond: (r.id = s.id) + -> Parallel Seq Scan on simple r + -> Parallel Hash + -> Parallel Seq Scan on simple s +(9 rows) + +select count(*) from simple r full outer join simple s using (id); + count +------- + 20000 +(1 row) + +rollback to settings; +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; @@ -869,8 +902,9 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); @@ -895,6 +929,31 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); 120000 (1 row) +rollback to settings; +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + QUERY PLAN +------------------------------------------------------------- + Finalize Aggregate + -> Gather + Workers Planned: 2 + -> Partial Aggregate + -> Parallel Hash Full Join + Hash Cond: ((0 - s.id) = r.id) + -> Parallel Seq Scan on simple s + -> Parallel Hash + -> Parallel Seq Scan on simple r +(9 rows) + +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + count +------- + 40000 +(1 row) + rollback to settings; -- exercise special code paths for huge tuples (note use of non-strict -- expression and left join required to get the detoasted tuple into diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 325068e9d23..01961d1ce6e 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -191,6 +191,8 @@ select original > 1 as initially_multibatch, final > original as increased_batch $$ select count(*) from simple r join simple s using (id); $$); +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); rollback to settings; -- The "bad" case: during execution we need to increase number of @@ -438,15 +440,24 @@ explain (costs off) select count(*) from simple r full outer join simple s using (id); rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); select count(*) from simple r full outer join simple s using (id); rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); +select count(*) from simple r full outer join simple s using (id); +rollback to settings; + +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; @@ -456,14 +467,24 @@ explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); +rollback to settings; + + -- exercise special code paths for huge tuples (note use of non-strict -- expression and left join required to get the detoasted tuple into -- the hash table) From 7ff139c5b0b1c37666b061408fab8070cb1c2b7a Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Fri, 14 Apr 2023 10:52:58 +1200 Subject: [PATCH 050/167] Fix PHJ match bit initialization. Hash join tuples reuse the HOT status bit to indicate match status during hash join execution. Correct reuse requires clearing the bit in all tuples. Serial hash join and parallel multi-batch hash join do so upon inserting the tuple into the hashtable. Single batch parallel hash join and batch 0 of unexpected multi-batch hash joins forgot to do this. It hadn't come up before because hashtable tuple match bits are only used for right and full outer joins and parallel ROJ and FOJ were unsupported. 11c2d6fdf5 introduced support for parallel ROJ/FOJ but neglected to ensure the match bits were reset. Author: Melanie Plageman Reported-by: Richard Guo Discussion: https://postgr.es/m/flat/CAMbWs48Nde1Mv%3DBJv6_vXmRKHMuHZm2Q_g4F6Z3_pn%2B3EV6BGQ%40mail.gmail.com --- src/backend/executor/nodeHash.c | 1 + src/test/regress/expected/join_hash.out | 37 +++++++++++++++++++++++++ src/test/regress/sql/join_hash.sql | 27 ++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index e3e1d5a6d87..d4a4fbc84f2 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2011,6 +2011,7 @@ ExecParallelHashTableInsert(HashJoinTable hashtable, /* Store the hash value in the HashJoinTuple header. */ hashTuple->hashvalue = hashvalue; memcpy(HJTUPLE_MINTUPLE(hashTuple), tuple, tuple->t_len); + HeapTupleHeaderClearMatch(HJTUPLE_MINTUPLE(hashTuple)); /* Push it onto the front of the bucket's list */ ExecParallelHashPushTuple(&hashtable->buckets.shared[bucketno], diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 250704efbd7..b1f780ff7b8 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -1031,6 +1031,43 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b (7 rows) rollback to settings; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +SAVEPOINT settings; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +CREATE TABLE hjtest_matchbits_t2(id int); +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +ROLLBACK TO settings; rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 01961d1ce6e..0858e14040f 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -539,6 +539,33 @@ rollback to settings; rollback; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +SAVEPOINT settings; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +CREATE TABLE hjtest_matchbits_t2(id int); +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +ROLLBACK TO settings; + +rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's From 2e6c773a5caacd7b14e1bb762a173f0d4f3197c5 Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Mon, 12 Jun 2023 12:19:46 +0900 Subject: [PATCH 051/167] Fix instability in regression test for Parallel Hash Full Join As reported by buildfarm member conchuela, one of the regression tests added by 558c9d7 is having some ordering issues. This commit adds an ORDER BY clause to make the output more stable for the problematic query. Fix suggested by Tom Lane. The plan of the query updated still uses a parallel hash full join. Author: Melanie Plageman Discussion: https://postgr.es/m/623596.1684541098@sss.pgh.pa.us --- src/test/regress/expected/join_hash.out | 3 ++- src/test/regress/sql/join_hash.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index b1f780ff7b8..28f4558ec04 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -1047,7 +1047,8 @@ INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't -- correctly emit the NULL-extended unmatching tuple in full hash join. UPDATE hjtest_matchbits_t2 set id = 2; -SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; id | id ----+---- 1 | diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 0858e14040f..0115489a6b9 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -555,7 +555,8 @@ INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't -- correctly emit the NULL-extended unmatching tuple in full hash join. UPDATE hjtest_matchbits_t2 set id = 2; -SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; -- Test serial full hash join. -- Resetting parallel_setup_cost should force a serial plan. -- Just to be safe, however, set enable_parallel_hash to off, as parallel full From e8f08e5f440526c8c4619fb76a6bf426902802c1 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 24 Mar 2026 13:00:49 +0800 Subject: [PATCH 052/167] Parallel Hash Full Join and Right Join PostgreSQL originally excluded FULL and RIGHT outer joins from parallel hash join because of deadlock hazards in the per-batch barrier protocol. PG 14 resolved this by introducing a dedicated PHJ_BATCH_SCAN phase: one elected worker emits unmatched inner-side rows after probing, while the others detach and move on. In CBDB, distributed execution adds a second dimension: after a full outer join the unmatched NULL-filled rows may come from any segment, so the result carries a HashedOJ locus rather than a plain Hashed locus. This change teaches the parallel planner about that: - FULL JOIN and RIGHT JOIN are now valid parallel join types in the distributed planner. Previously they were unconditionally rejected, forcing serial execution across all segments. - The HashedOJ locus produced by a parallel full join now carries parallel_workers, so operators above the join (aggregates, further joins) can remain parallel. - A crash that could occur when a parallel LASJ_NOTIN (NOT IN) join encountered NULL inner keys is fixed. The worker would exit early but the batch barrier, which was never attached to, would be touched on shutdown causing an assertion failure. Example plans (3 segments, parallel_workers=2): -- FULL JOIN: result locus is HashedOJ with Parallel Workers: 2 EXPLAIN(costs off, locus) SELECT count(*) FROM t1 FULL JOIN t2 USING (id); Finalize Aggregate Locus: Entry -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate Locus: HashedOJ Parallel Workers: 2 -> Parallel Hash Full Join Locus: HashedOJ Parallel Workers: 2 Hash Cond: (t1.id = t2.id) -> Parallel Seq Scan on t1 Locus: HashedWorkers -> Parallel Hash -> Parallel Seq Scan on t2 Locus: HashedWorkers -- RIGHT JOIN: when t1 is larger the planner hashes the smaller t2 -- and probes with t1; result locus HashedWorkers EXPLAIN(costs off, locus) SELECT count(*) FROM t1 RIGHT JOIN t2 USING (id); Finalize Aggregate Locus: Entry -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate Locus: HashedWorkers Parallel Workers: 2 -> Parallel Hash Right Join Locus: HashedWorkers Parallel Workers: 2 Hash Cond: (t1.id = t2.id) -> Parallel Seq Scan on t1 Locus: HashedWorkers -> Parallel Hash -> Parallel Seq Scan on t2 Locus: HashedWorkers Performance (3 segments x 2 parallel workers, 6M rows each, 50% overlap): FULL JOIN parallel: 4040 ms serial: 6347 ms speedup: 1.57x RIGHT JOIN parallel: 3039 ms serial: 5568 ms speedup: 1.83x --- src/backend/cdb/cdbpath.c | 5 +++-- src/backend/cdb/cdbpathlocus.c | 28 +++++++++++++++++++++------- src/backend/executor/nodeHash.c | 19 ++++++++++--------- src/backend/executor/nodeHashjoin.c | 6 +++--- src/include/cdb/cdbpathlocus.h | 4 ++-- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/backend/cdb/cdbpath.c b/src/backend/cdb/cdbpath.c index 9e3697a3b03..e9d7dac9895 100644 --- a/src/backend/cdb/cdbpath.c +++ b/src/backend/cdb/cdbpath.c @@ -3112,8 +3112,9 @@ cdbpath_motion_for_parallel_join(PlannerInfo *root, case JOIN_UNIQUE_INNER: case JOIN_RIGHT: case JOIN_FULL: - /* Join types are not supported in parallel yet. */ - goto fail; + outer.ok_to_replicate = false; + inner.ok_to_replicate = false; + break; case JOIN_DEDUP_SEMI: if (!enable_parallel_dedup_semi_join) goto fail; diff --git a/src/backend/cdb/cdbpathlocus.c b/src/backend/cdb/cdbpathlocus.c index 29930085429..dddae1aa64c 100644 --- a/src/backend/cdb/cdbpathlocus.c +++ b/src/backend/cdb/cdbpathlocus.c @@ -119,6 +119,11 @@ cdbpathlocus_equal(CdbPathLocus a, CdbPathLocus b) list_length(a.distkey) != list_length(b.distkey)) return false; + /* + * CBDB_PARALLEL: What if both a and b are HashedOJ with parallel workers > 0 ? + * Are they equal in practice? + */ + if ((CdbPathLocus_IsHashed(a) || CdbPathLocus_IsHashedOJ(a)) && (CdbPathLocus_IsHashed(b) || CdbPathLocus_IsHashedOJ(b))) return cdbpath_distkey_equal(a.distkey, b.distkey); @@ -544,7 +549,7 @@ cdbpathlocus_from_subquery(struct PlannerInfo *root, else { Assert(CdbPathLocus_IsHashedOJ(subpath->locus)); - CdbPathLocus_MakeHashedOJ(&locus, distkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&locus, distkeys, numsegments, subpath->locus.parallel_workers); } } else @@ -711,7 +716,7 @@ cdbpathlocus_pull_above_projection(struct PlannerInfo *root, CdbPathLocus_MakeHashedWorkers(&newlocus, newdistkeys, numsegments, locus.parallel_workers); } else - CdbPathLocus_MakeHashedOJ(&newlocus, newdistkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&newlocus, newdistkeys, numsegments, locus.parallel_workers); return newlocus; } else @@ -880,7 +885,7 @@ cdbpathlocus_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b) newdistkeys = lappend(newdistkeys, newdistkey); } - CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments); + CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments, 0 /* Both are 0 parallel here*/); } Assert(cdbpathlocus_is_valid(resultlocus)); return resultlocus; @@ -1236,8 +1241,14 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo Assert(cdbpathlocus_is_valid(a)); Assert(cdbpathlocus_is_valid(b)); - /* Do both input rels have same locus? */ - if (cdbpathlocus_equal(a, b)) + /* + * Do both input rels have same locus? + * CBDB_PARALLEL: for FULL JOIN, it could be different even both + * are same loucs. Because the NULL values could be on any segments + * after join. + */ + + if (jointype != JOIN_FULL && cdbpathlocus_equal(a, b)) return a; /* @@ -1412,8 +1423,9 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo * If inner is hashed workers, and outer is hashed. Join locus will be hashed. * If outer is hashed workers, and inner is hashed. Join locus will be hashed workers. * Seems we should just return outer locus anyway. + * Things changed since we have parallel full join now. */ - if (parallel_aware) + if (parallel_aware && jointype != JOIN_FULL) return a; numsegments = CdbPathLocus_NumSegments(a); @@ -1469,7 +1481,9 @@ cdbpathlocus_parallel_join(JoinType jointype, CdbPathLocus a, CdbPathLocus b, bo newdistkeys = lappend(newdistkeys, newdistkey); } - CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments); + Assert(CdbPathLocus_NumParallelWorkers(a) == CdbPathLocus_NumParallelWorkers(b)); + + CdbPathLocus_MakeHashedOJ(&resultlocus, newdistkeys, numsegments, CdbPathLocus_NumParallelWorkers(a)); } Assert(cdbpathlocus_is_valid(resultlocus)); return resultlocus; diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index d4a4fbc84f2..e59a7c7ccc3 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -2408,11 +2408,11 @@ ExecParallelPrepHashTableForUnmatched(HashJoinState *hjstate) int curbatch = hashtable->curbatch; ParallelHashJoinBatch *batch = hashtable->batches[curbatch].shared; - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE); + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING); /* * It would not be deadlock-free to wait on the batch barrier, because it - * is in PHJ_BATCH_PROBE phase, and thus processes attached to it have + * is in PHJ_BATCH_PROBING phase, and thus processes attached to it have * already emitted tuples. Therefore, we'll hold a wait-free election: * only one process can continue to the next phase, and all others detach * from this batch. They can still go any work on other batches, if there @@ -3979,12 +3979,12 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) sts_end_parallel_scan(hashtable->batches[curbatch].inner_tuples); sts_end_parallel_scan(hashtable->batches[curbatch].outer_tuples); - /* After attaching we always get at least to PHJ_BATCH_PROBE. */ - Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE || + /* After attaching we always get at least to PHJ_BATCH_PROBING. */ + Assert(BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING || BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_SCAN); /* - * If we're abandoning the PHJ_BATCH_PROBE phase early without having + * If we're abandoning the PHJ_BATCH_PROBING phase early without having * reached the end of it, it means the plan doesn't want any more * tuples, and it is happy to abandon any tuples buffered in this * process's subplans. For correctness, we can't allow any process to @@ -3999,13 +3999,13 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) * If phs_lasj_has_null is true, that means we have found null when building hash table, * there were no batches to detach. */ - if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING && !hashtable->parallel_state->phs_lasj_has_null && /* CBDB_PARALLEL */ !hashtable->batches[curbatch].outer_eof) { /* * This flag may be written to by multiple backends during - * PHJ_BATCH_PROBE phase, but will only be read in PHJ_BATCH_SCAN + * PHJ_BATCH_PROBING phase, but will only be read in PHJ_BATCH_SCAN * phase so requires no extra locking. */ batch->skip_unmatched = true; @@ -4016,10 +4016,11 @@ ExecHashTableDetachBatch(HashJoinTable hashtable) * the PHJ_BATCH_SCAN phase just to maintain the invariant that * freeing happens in PHJ_BATCH_FREE, but that'll be wait-free. */ - if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBE && + if (BarrierPhase(&batch->batch_barrier) == PHJ_BATCH_PROBING && !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */) attached = BarrierArriveAndDetachExceptLast(&batch->batch_barrier); - if (attached && BarrierArriveAndDetach(&batch->batch_barrier)) + if (attached && !hashtable->parallel_state->phs_lasj_has_null /* CBDB_PARALLEL */ && + BarrierArriveAndDetach(&batch->batch_barrier)) { /* * We are not longer attached to the batch barrier, but we're the diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 53bb29be7af..9981ed8f7ae 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -84,7 +84,7 @@ * PHJ_BATCH_ELECT -- initial state * PHJ_BATCH_ALLOCATE* -- one allocates buckets * PHJ_BATCH_LOAD -- all load the hash table from disk - * PHJ_BATCH_PROBE -- all probe + * PHJ_BATCH_PROBING -- all probe * PHJ_BATCH_SCAN* -- one does full/right unmatched scan * PHJ_BATCH_FREE* -- one frees memory * @@ -103,7 +103,7 @@ * to a barrier, unless the barrier has reached a phase that means that no * process will wait on it again. We emit tuples while attached to the build * barrier in phase PHJ_BUILD_RUN, and to a per-batch barrier in phase - * PHJ_BATCH_PROBE. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN + * PHJ_BATCH_PROBING. These are advanced to PHJ_BUILD_FREE and PHJ_BATCH_SCAN * respectively without waiting, using BarrierArriveAndDetach() and * BarrierArriveAndDetachExceptLast() respectively. The last to detach * receives a different return value so that it knows that it's safe to @@ -1566,7 +1566,7 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * since that phase is already underway (the thing we * can't do under current deadlock-avoidance rules is wait * for others to arrive at PHJ_BATCH_SCAN, because - * PHJ_BATCH_PROBE emits tuples, but in this case we just + * PHJ_BATCH_PROBING emits tuples, but in this case we just * got here without waiting). That is not yet done. For * now, we just detach and go around again. We have to * use ExecHashTableDetachBatch() because there's a small diff --git a/src/include/cdb/cdbpathlocus.h b/src/include/cdb/cdbpathlocus.h index 0f71ba55dfb..9f5a8227e68 100644 --- a/src/include/cdb/cdbpathlocus.h +++ b/src/include/cdb/cdbpathlocus.h @@ -292,13 +292,13 @@ typedef struct CdbPathLocus _locus->parallel_workers = (parallel_workers_); \ Assert(cdbpathlocus_is_valid(*_locus)); \ } while (0) -#define CdbPathLocus_MakeHashedOJ(plocus, distkey_, numsegments_) \ +#define CdbPathLocus_MakeHashedOJ(plocus, distkey_, numsegments_, parallel_workers_) \ do { \ CdbPathLocus *_locus = (plocus); \ _locus->locustype = CdbLocusType_HashedOJ; \ _locus->numsegments = (numsegments_); \ _locus->distkey = (distkey_); \ - _locus->parallel_workers = 0; \ + _locus->parallel_workers = (parallel_workers_); \ Assert(cdbpathlocus_is_valid(*_locus)); \ } while (0) #define CdbPathLocus_MakeHashedWorkers(plocus, distkey_, numsegments_, parallel_workers_) \ From 5045e37e013d5e87dff96284e09e5941c52c9a50 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 24 Mar 2026 16:33:27 +0800 Subject: [PATCH 053/167] tests: add Parallel Hash Full/Right Join regression cases cbdb_parallel.sql: add a new test block covering: - Parallel Hash Full Join (HashedWorkers FULL JOIN HashedWorkers produces HashedOJ with parallel_workers=2) - Parallel Hash Right Join (pj_t1 is 3x larger than pj_t2, so the planner hashes the smaller pj_t2 and probes with pj_t1; result locus HashedWorkers) - Correctness checks: count(*) matches serial execution - Locus propagation: HashedOJ(parallel) followed by INNER JOIN produces HashedOJ; followed by FULL JOIN produces HashedOJ join_hash.sql/out: CBDB-specific adaptations for the upstream parallel full join test -- disable parallel mode for tests that require serial plans, fix SAVEPOINT inside a parallel worker context, and update expected output to match CBDB plan shapes. --- .../pax_storage/expected/cbdb_parallel.out | 183 ++++++--- src/test/regress/expected/cbdb_parallel.out | 370 +++++++++++++----- src/test/regress/expected/join_hash.out | 83 ++-- .../regress/expected/join_hash_optimizer.out | 204 +++++++--- src/test/regress/sql/cbdb_parallel.sql | 50 +++ src/test/regress/sql/join_hash.sql | 6 + 6 files changed, 668 insertions(+), 228 deletions(-) diff --git a/contrib/pax_storage/expected/cbdb_parallel.out b/contrib/pax_storage/expected/cbdb_parallel.out index db583090026..ec6ceba7e3c 100644 --- a/contrib/pax_storage/expected/cbdb_parallel.out +++ b/contrib/pax_storage/expected/cbdb_parallel.out @@ -41,13 +41,29 @@ set gp_appendonly_insert_files = 4; begin; set local enable_parallel = on; create table test_131_ao1(x int, y int) using ao_row with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao2(x int, y int) using ao_row with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao3(x int, y int) using ao_row with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_ao4(x int, y int) using ao_row with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco1(x int, y int) using ao_column with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco2(x int, y int) using ao_column with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco3(x int, y int) using ao_column with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table test_131_aoco4(x int, y int) using ao_column with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. select relname, reloptions from pg_catalog.pg_class where relname like 'test_131_ao%'; relname | reloptions ----------------+---------------------- @@ -155,8 +171,14 @@ explain(locus, costs off) select count(*) from test_131_aoco3, test_131_aoco4 wh abort; create table ao1(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table ao2(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table aocs1(x int, y int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. begin; -- encourage use of parallel plans set local min_parallel_table_scan_size = 0; @@ -367,6 +389,8 @@ abort; begin; set local max_parallel_workers_per_gather = 2; create table t1(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table rt1(a int, b int) with(parallel_workers=2) distributed replicated; create table rt2(a int, b int) distributed replicated; create table rt3(a int, b int) distributed replicated; @@ -599,6 +623,8 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -606,8 +632,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 6 | 7 | 5 | 6 | 6 | 7 7 | 8 | 6 | 7 | 7 | 8 10 | 11 | 9 | 10 | 10 | 11 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 (19 rows) -- parallel hash join @@ -650,13 +674,6 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 5 | 6 | 6 | 7 - 6 | 7 | 6 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 9 | 10 | 10 | 11 - 10 | 11 | 10 | 10 | 10 | 11 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 2 | 3 | 3 | 4 3 | 4 | 3 | 3 | 3 | 4 @@ -669,6 +686,13 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 9 | 10 | 8 | 9 | 9 | 10 1 | 2 | 1 | 1 | 1 | 2 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 5 | 6 | 6 | 7 + 6 | 7 | 6 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 9 | 10 | 10 | 11 + 10 | 11 | 10 | 10 | 10 | 11 (19 rows) -- @@ -702,6 +726,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt3 on select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -712,8 +738,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -779,6 +803,8 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; (19 rows) create table t2(a int, b int) with(parallel_workers=0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table rt4(a int, b int) with(parallel_workers=2) distributed replicated; insert into t2 select i, i+1 from generate_series(1, 10) i; insert into rt4 select i, i+1 from generate_series(1, 10000) i; @@ -788,16 +814,16 @@ set local enable_parallel = off; select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 - 6 | 5 | 5 - 7 | 6 | 6 - 10 | 9 | 9 - 11 | 10 | 10 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 + 2 | 1 | 1 + 6 | 5 | 5 + 7 | 6 | 6 + 10 | 9 | 9 + 11 | 10 | 10 (10 rows) set local enable_parallel = on; @@ -828,19 +854,21 @@ explain(locus, costs off) select * from rt4 join t2 using(b); select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 + 6 | 5 | 5 + 7 | 6 | 6 + 10 | 9 | 9 + 11 | 10 | 10 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 - 6 | 5 | 5 - 7 | 6 | 6 - 10 | 9 | 9 - 11 | 10 | 10 + 2 | 1 | 1 (10 rows) create table t3(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t3 select i, i+1 from generate_series(1, 9000) i; analyze t3; set local enable_parallel = off; @@ -919,10 +947,10 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 2 | 3 | 3 4 | 5 | 5 + 3 | 4 | 4 5 | 6 | 6 (5 rows) @@ -931,11 +959,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 3 | 4 | 4 - 1 | 2 | 2 - 4 | 5 | 5 5 | 6 | 6 + 4 | 5 | 5 + 1 | 2 | 2 + 2 | 3 | 3 (5 rows) abort; @@ -976,11 +1004,11 @@ explain(locus, costs off) select * from t_replica_workers_2 right join t_random_ select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- - 5 | 6 | 6 1 | 2 | 2 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 + 5 | 6 | 6 (5 rows) -- non parallel results @@ -1028,14 +1056,14 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker Locus: Strewn Parallel Workers: 2 Optimizer: Postgres query optimizer -(16 rows) +(15 rows) select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1045,9 +1073,9 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1059,7 +1087,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(b int, a int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10) i; insert into t2 select i, i+1 from generate_series(1, 5) i; analyze t1; @@ -1071,17 +1103,17 @@ explain(costs off) select * from t1 right join t2 on t1.b = t2.a; QUERY PLAN ------------------------------------------------------------------ Gather Motion 9:1 (slice1; segments: 9) - -> Parallel Hash Left Join - Hash Cond: (t2.a = t1.b) - -> Redistribute Motion 6:9 (slice2; segments: 6) - Hash Key: t2.a + -> Parallel Hash Right Join + Hash Cond: (t1.b = t2.a) + -> Redistribute Motion 9:9 (slice2; segments: 9) + Hash Key: t1.b Hash Module: 3 - -> Parallel Seq Scan on t2 + -> Parallel Seq Scan on t1 -> Parallel Hash - -> Redistribute Motion 9:9 (slice3; segments: 9) - Hash Key: t1.b + -> Redistribute Motion 6:9 (slice3; segments: 6) + Hash Key: t2.a Hash Module: 3 - -> Parallel Seq Scan on t1 + -> Parallel Seq Scan on t2 Optimizer: Postgres query optimizer (13 rows) @@ -1091,7 +1123,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(a int, b int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i%10, i from generate_series(1, 5) i; insert into t1 values (100000); insert into t2 select i%10, i from generate_series(1, 100000) i; @@ -1100,34 +1136,34 @@ analyze t2; set local enable_parallel = on; -- parallel hash join with shared table, SinglQE as outer partial path. explain(locus, costs off) select * from (select count(*) as a from t2) t2 left join t1 on t1.a = t2.a; - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Gather Motion 6:1 (slice1; segments: 6) Locus: Entry - -> Parallel Hash Left Join - Locus: Hashed + -> Parallel Hash Right Join + Locus: HashedWorkers Parallel Workers: 2 - Hash Cond: ((count(*)) = t1.a) - -> Redistribute Motion 1:6 (slice2; segments: 1) - Locus: Hashed + Hash Cond: (t1.a = (count(*))) + -> Parallel Seq Scan on t1 + Locus: HashedWorkers Parallel Workers: 2 - Hash Key: (count(*)) - Hash Module: 3 - -> Finalize Aggregate - Locus: SingleQE - -> Gather Motion 6:1 (slice3; segments: 6) - Locus: SingleQE - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t2 - Locus: HashedWorkers - Parallel Workers: 2 -> Parallel Hash Locus: Hashed - -> Parallel Seq Scan on t1 - Locus: HashedWorkers + -> Redistribute Motion 1:6 (slice2; segments: 1) + Locus: Hashed Parallel Workers: 2 + Hash Key: (count(*)) + Hash Module: 3 + -> Finalize Aggregate + Locus: SingleQE + -> Gather Motion 6:1 (slice3; segments: 6) + Locus: SingleQE + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t2 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (27 rows) @@ -1323,12 +1359,18 @@ begin; create table rt1(a int, b int) distributed replicated; create table rt2(a int, b int) with (parallel_workers = 0) distributed replicated; create table t1(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(a int, b int) with (parallel_workers = 0); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10000) i; insert into t2 select i, i+1 from generate_series(1, 10000) i; insert into rt1 select i, i+1 from generate_series(1, 10000) i; insert into rt2 select i, i+1 from generate_series(1, 10000) i; CREATE TABLE sq1 AS SELECT a, b FROM t1 WHERE gp_segment_id = 0; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. set local optimizer=off; set local enable_parallel=on; set local min_parallel_table_scan_size to 0; @@ -1385,7 +1427,7 @@ explain (locus, costs off) select * from rt1 union all select * from t1; -> Result Locus: Strewn Parallel Workers: 2 - One-Time Filter: (gp_execution_segment() = 1) + One-Time Filter: (gp_execution_segment() = 0) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 2 @@ -1409,7 +1451,7 @@ explain (locus, costs off) select * from rt1 union all select * from t2; -> Result Locus: Strewn Parallel Workers: 2 - One-Time Filter: (gp_execution_segment() = 1) + One-Time Filter: (gp_execution_segment() = 0) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 2 @@ -1482,6 +1524,8 @@ abort; -- begin; create table t1(c1 int, c2 int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 100000) i; analyze t1; set local optimizer = off; @@ -1549,6 +1593,8 @@ abort; -- begin; create table t1(c1 int, c2 int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 100000) i; analyze t1; set local optimizer = off; @@ -1768,6 +1814,8 @@ set local optimizer = off; set local enable_parallel = on; -- ao table create table ao (a INT, b INT) using ao_row; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into ao select i as a, i as b from generate_series(1, 100) AS i; alter table ao set (parallel_workers = 2); explain(costs off) select count(*) from ao; @@ -1789,6 +1837,8 @@ select count(*) from ao; alter table ao reset (parallel_workers); -- aocs table create table aocs (a INT, b INT) using ao_column; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into aocs select i as a, i as b from generate_series(1, 100) AS i; alter table aocs set (parallel_workers = 2); explain(costs off) select count(*) from aocs; @@ -1862,9 +1912,14 @@ select * from abort; begin; create table pagg_tab (a int, b int, c text, d int) partition by list(c); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table pagg_tab_p1 partition of pagg_tab for values in ('0000', '0001', '0002', '0003', '0004'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p2 partition of pagg_tab for values in ('0005', '0006', '0007', '0008'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p3 partition of pagg_tab for values in ('0009', '0010', '0011'); +NOTICE: table has parent, setting distribution columns to match parent table insert into pagg_tab select i % 20, i % 30, to_char(i % 12, 'FM0000'), i % 30 from generate_series(0, 2999) i; analyze pagg_tab; set local enable_parallel to off; @@ -1939,7 +1994,11 @@ abort; -- begin; create table t1(a int, b int) with(parallel_workers=3); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2(b int, a int) with(parallel_workers=2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 select i, i+1 from generate_series(1, 10) i; insert into t2 select i, i+1 from generate_series(1, 5) i; analyze t1; @@ -2329,6 +2388,8 @@ abort; -- prepare, execute locus is null begin; create table t1(c1 int, c2 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze t1; prepare t1_count(integer) as select count(*) from t1; explain(locus, costs off) execute t1_count(1); diff --git a/src/test/regress/expected/cbdb_parallel.out b/src/test/regress/expected/cbdb_parallel.out index 35e90eebfa1..af975de50f4 100644 --- a/src/test/regress/expected/cbdb_parallel.out +++ b/src/test/regress/expected/cbdb_parallel.out @@ -112,8 +112,8 @@ set local enable_parallel_dedup_semi_reverse_join = on; set local enable_parallel_dedup_semi_join = on; explain (costs off) select sum(foo.a) from foo where exists (select 1 from bar where foo.a = bar.b); - QUERY PLAN ------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate @@ -1032,6 +1032,15 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 6 | 6 | 6 | 7 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 10 | 10 | 10 | 11 + 6 | 7 | 5 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 10 | 11 | 9 | 10 | 10 | 11 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1042,15 +1051,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 6 | 6 | 6 | 7 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 10 | 10 | 10 | 11 - 6 | 7 | 5 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 10 | 11 | 9 | 10 | 10 | 11 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 (19 rows) -- parallel hash join @@ -1093,13 +1093,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt2 on select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- - 5 | 6 | 5 | 5 | 5 | 6 - 6 | 7 | 5 | 6 | 6 | 7 - 6 | 7 | 6 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 9 | 10 | 9 | 9 | 9 | 10 - 10 | 11 | 9 | 10 | 10 | 11 - 10 | 11 | 10 | 10 | 10 | 11 + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 2 | 3 | 3 | 4 3 | 4 | 3 | 3 | 3 | 4 @@ -1110,8 +1105,13 @@ select * from rt1 join t1 on rt1.a = t1.b join rt2 on rt2.a = t1.b; 8 | 9 | 7 | 8 | 8 | 9 8 | 9 | 8 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 + 5 | 6 | 5 | 5 | 5 | 6 + 6 | 7 | 5 | 6 | 6 | 7 + 6 | 7 | 6 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 9 | 10 | 9 | 9 | 9 | 10 + 10 | 11 | 9 | 10 | 10 | 11 + 10 | 11 | 10 | 10 | 10 | 11 (19 rows) -- @@ -1145,6 +1145,8 @@ explain(locus, costs off) select * from rt1 join t1 on rt1.a = t1.b join rt3 on select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- + 1 | 2 | 1 | 1 | 1 | 2 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1155,8 +1157,6 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 - 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 @@ -1201,14 +1201,11 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; a | b | a | b | a | b ----+----+----+----+----+---- 1 | 2 | 1 | 1 | 1 | 2 - 2 | 3 | 1 | 2 | 2 | 3 5 | 6 | 5 | 5 | 5 | 6 6 | 7 | 6 | 6 | 6 | 7 9 | 10 | 9 | 9 | 9 | 10 10 | 11 | 10 | 10 | 10 | 11 - 6 | 7 | 5 | 6 | 6 | 7 - 7 | 8 | 6 | 7 | 7 | 8 - 10 | 11 | 9 | 10 | 10 | 11 + 2 | 3 | 1 | 2 | 2 | 3 2 | 3 | 2 | 2 | 2 | 3 3 | 4 | 3 | 3 | 3 | 4 4 | 5 | 4 | 4 | 4 | 5 @@ -1219,6 +1216,9 @@ select * from rt1 join t1 on rt1.a = t1.b join rt3 on rt3.a = t1.b; 5 | 6 | 4 | 5 | 5 | 6 8 | 9 | 7 | 8 | 8 | 9 9 | 10 | 8 | 9 | 9 | 10 + 6 | 7 | 5 | 6 | 6 | 7 + 7 | 8 | 6 | 7 | 7 | 8 + 10 | 11 | 9 | 10 | 10 | 11 (19 rows) create table t2(a int, b int) with(parallel_workers=0); @@ -1271,12 +1271,12 @@ explain(locus, costs off) select * from rt4 join t2 using(b); select * from rt4 join t2 using(b); b | a | a ----+----+---- - 2 | 1 | 1 3 | 2 | 2 4 | 3 | 3 5 | 4 | 4 8 | 7 | 7 9 | 8 | 8 + 2 | 1 | 1 6 | 5 | 5 7 | 6 | 6 10 | 9 | 9 @@ -1362,9 +1362,9 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1374,9 +1374,9 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_0 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 3 | 4 | 4 1 | 2 | 2 + 3 | 4 | 4 + 2 | 3 | 3 4 | 5 | 5 5 | 6 | 6 (5 rows) @@ -1419,9 +1419,9 @@ explain(locus, costs off) select * from t_replica_workers_2 right join t_random_ select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- + 2 | 3 | 3 5 | 6 | 6 1 | 2 | 2 - 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 (5 rows) @@ -1431,11 +1431,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 right join t_random_workers_2 using(a); a | b | b ---+---+--- + 5 | 6 | 6 1 | 2 | 2 - 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 - 5 | 6 | 6 + 2 | 3 | 3 (5 rows) abort; @@ -1471,13 +1471,13 @@ explain(locus, costs off) select * from t_replica_workers_2 join t_random_worker Locus: Strewn Parallel Workers: 2 Optimizer: Postgres query optimizer -(16 rows) +(15 rows) select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 1 | 2 | 2 + 2 | 3 | 3 3 | 4 | 4 4 | 5 | 5 5 | 6 | 6 @@ -1488,11 +1488,11 @@ set local enable_parallel=false; select * from t_replica_workers_2 join t_random_workers_2 using(a); a | b | b ---+---+--- - 2 | 3 | 3 - 1 | 2 | 2 3 | 4 | 4 4 | 5 | 5 5 | 6 | 6 + 1 | 2 | 2 + 2 | 3 | 3 (5 rows) abort; @@ -1510,28 +1510,28 @@ analyze t1; analyze rt1; set local enable_parallel = on; explain(locus, costs off) select * from (select count(*) as a from t1) t1 left join rt1 on rt1.a = t1.a; - QUERY PLAN ------------------------------------------------------- - Parallel Hash Left Join + QUERY PLAN +------------------------------------------------------------ + Parallel Hash Right Join Locus: Entry - Hash Cond: ((count(*)) = rt1.a) - -> Finalize Aggregate + Hash Cond: (rt1.a = (count(*))) + -> Gather Motion 2:1 (slice1; segments: 2) Locus: Entry - -> Gather Motion 6:1 (slice1; segments: 6) - Locus: Entry - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t1 - Locus: HashedWorkers - Parallel Workers: 2 + -> Parallel Seq Scan on rt1 + Locus: SegmentGeneralWorkers + Parallel Workers: 2 -> Parallel Hash Locus: Entry - -> Gather Motion 2:1 (slice2; segments: 2) + -> Finalize Aggregate Locus: Entry - -> Parallel Seq Scan on rt1 - Locus: SegmentGeneralWorkers - Parallel Workers: 2 + -> Gather Motion 6:1 (slice2; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t1 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (21 rows) @@ -1661,17 +1661,17 @@ explain(costs off) select * from t1 right join t2 on t1.b = t2.a; QUERY PLAN ------------------------------------------------------------------ Gather Motion 9:1 (slice1; segments: 9) - -> Parallel Hash Left Join - Hash Cond: (t2.a = t1.b) - -> Redistribute Motion 6:9 (slice2; segments: 6) - Hash Key: t2.a + -> Parallel Hash Right Join + Hash Cond: (t1.b = t2.a) + -> Redistribute Motion 9:9 (slice2; segments: 9) + Hash Key: t1.b Hash Module: 3 - -> Parallel Seq Scan on t2 + -> Parallel Seq Scan on t1 -> Parallel Hash - -> Redistribute Motion 9:9 (slice3; segments: 9) - Hash Key: t1.b + -> Redistribute Motion 6:9 (slice3; segments: 6) + Hash Key: t2.a Hash Module: 3 - -> Parallel Seq Scan on t1 + -> Parallel Seq Scan on t2 Optimizer: Postgres query optimizer (13 rows) @@ -1690,34 +1690,34 @@ analyze t2; set local enable_parallel = on; -- parallel hash join with shared table, SinglQE as outer partial path. explain(locus, costs off) select * from (select count(*) as a from t2) t2 left join t1 on t1.a = t2.a; - QUERY PLAN ------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Gather Motion 6:1 (slice1; segments: 6) Locus: Entry - -> Parallel Hash Left Join - Locus: Hashed + -> Parallel Hash Right Join + Locus: HashedWorkers Parallel Workers: 2 - Hash Cond: ((count(*)) = t1.a) - -> Redistribute Motion 1:6 (slice2; segments: 1) - Locus: Hashed + Hash Cond: (t1.a = (count(*))) + -> Parallel Seq Scan on t1 + Locus: HashedWorkers Parallel Workers: 2 - Hash Key: (count(*)) - Hash Module: 3 - -> Finalize Aggregate - Locus: SingleQE - -> Gather Motion 6:1 (slice3; segments: 6) - Locus: SingleQE - -> Partial Aggregate - Locus: HashedWorkers - Parallel Workers: 2 - -> Parallel Seq Scan on t2 - Locus: HashedWorkers - Parallel Workers: 2 -> Parallel Hash Locus: Hashed - -> Parallel Seq Scan on t1 - Locus: HashedWorkers + -> Redistribute Motion 1:6 (slice2; segments: 1) + Locus: Hashed Parallel Workers: 2 + Hash Key: (count(*)) + Hash Module: 3 + -> Finalize Aggregate + Locus: SingleQE + -> Gather Motion 6:1 (slice3; segments: 6) + Locus: SingleQE + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Seq Scan on t2 + Locus: HashedWorkers + Parallel Workers: 2 Optimizer: Postgres query optimizer (27 rows) @@ -1975,7 +1975,7 @@ explain (locus, costs off) select * from rt1 union all select * from t1; -> Result Locus: Strewn Parallel Workers: 3 - One-Time Filter: (gp_execution_segment() = 0) + One-Time Filter: (gp_execution_segment() = 1) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 3 @@ -1999,7 +1999,7 @@ explain (locus, costs off) select * from rt1 union all select * from t2; -> Result Locus: Strewn Parallel Workers: 3 - One-Time Filter: (gp_execution_segment() = 0) + One-Time Filter: (gp_execution_segment() = 1) -> Parallel Seq Scan on rt1 Locus: SegmentGeneralWorkers Parallel Workers: 3 @@ -2296,8 +2296,8 @@ analyze t1; analyze t2; analyze t3_null; explain(costs off) select sum(t1.c1) from t1 where c1 not in (select c2 from t2); - QUERY PLAN ------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 6:1 (slice1; segments: 6) -> Partial Aggregate @@ -2317,8 +2317,8 @@ select sum(t1.c1) from t1 where c1 not in (select c2 from t2); (1 row) explain(costs off) select * from t1 where c1 not in (select c2 from t3_null); - QUERY PLAN ------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------- Gather Motion 6:1 (slice1; segments: 6) -> Parallel Hash Left Anti Semi (Not-In) Join Hash Cond: (t1.c1 = t3_null.c2) @@ -2457,8 +2457,11 @@ abort; begin; create table pagg_tab (a int, b int, c text, d int) partition by list(c); create table pagg_tab_p1 partition of pagg_tab for values in ('0000', '0001', '0002', '0003', '0004'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p2 partition of pagg_tab for values in ('0005', '0006', '0007', '0008'); +NOTICE: table has parent, setting distribution columns to match parent table create table pagg_tab_p3 partition of pagg_tab for values in ('0009', '0010', '0011'); +NOTICE: table has parent, setting distribution columns to match parent table insert into pagg_tab select i % 20, i % 30, to_char(i % 12, 'FM0000'), i % 30 from generate_series(0, 2999) i; analyze pagg_tab; set local enable_parallel to off; @@ -2972,7 +2975,7 @@ create table t2_anti(a int, b int) with(parallel_workers=2) distributed by (b); insert into t2_anti values(generate_series(5, 10)); explain(costs off, verbose) select t1_anti.a, t1_anti.b from t1_anti left join t2_anti on t1_anti.a = t2_anti.a where t2_anti.a is null; - QUERY PLAN + QUERY PLAN ------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) Output: t1_anti.a, t1_anti.b @@ -3068,8 +3071,8 @@ select t1_anti.a, t1_anti.b from t1_anti left join t2_anti on t1_anti.a = t2_ant ---+--- 3 | 4 | - 1 | 2 | + 1 | (4 rows) abort; @@ -3098,7 +3101,7 @@ insert into t_distinct_0 select * from t_distinct_0; analyze t_distinct_0; explain(costs off) select distinct a from t_distinct_0; - QUERY PLAN + QUERY PLAN ------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) -> HashAggregate @@ -3232,8 +3235,6 @@ select distinct a, b from t_distinct_0; drop table if exists t_distinct_1; NOTICE: table "t_distinct_1" does not exist, skipping create table t_distinct_1(a int, b int) using ao_column; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t_distinct_1 select * from t_distinct_0; analyze t_distinct_1; set enable_parallel = off; @@ -3520,10 +3521,7 @@ WHERE e.salary > ( -- Test https://github.com/apache/cloudberry/issues/1376 -- create table t1(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table t2 (like t1); -NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table set gp_cte_sharing = on; explain(locus, costs off) with x as (select a, count(*) as b from t1 group by a union all @@ -3571,8 +3569,184 @@ explain(locus, costs off) with x as reset gp_cte_sharing; reset enable_parallel; reset min_parallel_table_scan_size; +-- +-- Parallel Hash Full/Right Join +-- +begin; +create table pj_t1(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t2(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t3(id int, v int) with(parallel_workers=0) distributed by (id); +-- pj_t1 is 3x larger than pj_t2 so the planner hashes the smaller pj_t2 +-- and probes with pj_t1, producing a genuine Parallel Hash Right Join plan. +insert into pj_t1 select i, i from generate_series(1,30000)i; +insert into pj_t2 select i, i from generate_series(25001,35000)i; +insert into pj_t3 select i, i from generate_series(1,10000)i; +analyze pj_t1; +analyze pj_t2; +analyze pj_t3; +set local enable_parallel = on; +set local min_parallel_table_scan_size = 0; +-- 12_P_12_10: Parallel Hash Full Join: HashedWorkers FULL JOIN HashedWorkers -> HashedOJ(parallel) +explain(costs off, locus) +select count(*) from pj_t1 full join pj_t2 using (id); + QUERY PLAN +---------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 6:1 (slice1; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + Parallel Workers: 2 + -> Parallel Hash Full Join + Locus: HashedOJ + Parallel Workers: 2 + Hash Cond: (pj_t1.id = pj_t2.id) + -> Parallel Seq Scan on pj_t1 + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash + Locus: Hashed + -> Parallel Seq Scan on pj_t2 + Locus: HashedWorkers + Parallel Workers: 2 + Optimizer: Postgres query optimizer +(20 rows) + +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 full join pj_t2 using (id); + count +------- + 35000 +(1 row) + +set local enable_parallel = on; +select count(*) from pj_t1 full join pj_t2 using (id); + count +------- + 35000 +(1 row) + +-- Parallel Hash Right Join: pj_t1 (30K) is larger, so the planner hashes the smaller pj_t2 +-- (10K) as the build side and probes with pj_t1; result locus HashedWorkers(parallel) +explain(costs off, locus) +select count(*) from pj_t1 right join pj_t2 using (id); + QUERY PLAN +---------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 6:1 (slice1; segments: 6) + Locus: Entry + -> Partial Aggregate + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash Right Join + Locus: HashedWorkers + Parallel Workers: 2 + Hash Cond: (pj_t1.id = pj_t2.id) + -> Parallel Seq Scan on pj_t1 + Locus: HashedWorkers + Parallel Workers: 2 + -> Parallel Hash + Locus: Hashed + -> Parallel Seq Scan on pj_t2 + Locus: HashedWorkers + Parallel Workers: 2 + Optimizer: Postgres query optimizer +(20 rows) + +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 right join pj_t2 using (id); + count +------- + 10000 +(1 row) + +set local enable_parallel = on; +select count(*) from pj_t1 right join pj_t2 using (id); + count +------- + 10000 +(1 row) + +-- Locus propagation: HashedOJ(parallel) followed by INNER JOIN with Hashed(serial) +-- The full join result (HashedOJ,parallel=2) is joined with pj_t3 (Hashed,serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj inner join pj_t3 using (id); + QUERY PLAN +--------------------------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 3:1 (slice1; segments: 3) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + -> Hash Join + Locus: HashedOJ + Hash Cond: (COALESCE(pj_t1.id, pj_t2.id) = pj_t3.id) + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (pj_t1.id = pj_t2.id) + -> Seq Scan on pj_t1 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t2 + Locus: Hashed + -> Hash + Locus: Replicated + -> Broadcast Motion 3:3 (slice2; segments: 3) + Locus: Replicated + -> Seq Scan on pj_t3 + Locus: Hashed + Optimizer: Postgres query optimizer +(25 rows) + +-- Locus propagation: HashedOJ(parallel) followed by FULL JOIN with Hashed(serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj full join pj_t3 using (id); + QUERY PLAN +-------------------------------------------------------------------------- + Finalize Aggregate + Locus: Entry + -> Gather Motion 3:1 (slice1; segments: 3) + Locus: Entry + -> Partial Aggregate + Locus: HashedOJ + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (COALESCE(pj_t1.id, pj_t2.id) = pj_t3.id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Locus: Hashed + Hash Key: COALESCE(pj_t1.id, pj_t2.id) + -> Hash Full Join + Locus: HashedOJ + Hash Cond: (pj_t1.id = pj_t2.id) + -> Seq Scan on pj_t1 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t2 + Locus: Hashed + -> Hash + Locus: Hashed + -> Seq Scan on pj_t3 + Locus: Hashed + Optimizer: Postgres query optimizer +(26 rows) + +abort; -- start_ignore drop schema test_parallel cascade; +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to table t_distinct_0 +drop cascades to table t_distinct_1 +drop cascades to table departments +drop cascades to table employees +drop cascades to table t1 +drop cascades to table t2 -- end_ignore reset gp_appendonly_insert_files; reset force_parallel_mode; diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 28f4558ec04..e5f74c18d28 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -10,6 +10,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or -- buckets) will be required because it can vary, but we can in some @@ -58,12 +61,16 @@ $$; -- estimated size. create table simple as select generate_series(1, 60000) AS id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table simple set (parallel_workers = 2); analyze simple; -- Make a relation whose size we will under-estimate. We want stats -- to say 1000 rows, but actually there are 20,000 rows. create table bigger_than_it_looks as select generate_series(1, 60000) as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table bigger_than_it_looks set (autovacuum_enabled = 'false'); WARNING: autovacuum is not supported in Cloudberry alter table bigger_than_it_looks set (parallel_workers = 2); @@ -73,6 +80,8 @@ update pg_class set reltuples = 1000 where relname = 'bigger_than_it_looks'; -- kind of skew that breaks our batching scheme. We want stats to say -- 2 rows, but actually there are 20,000 rows with the same key. create table extremely_skewed (id int, t text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table extremely_skewed set (autovacuum_enabled = 'false'); WARNING: autovacuum is not supported in Cloudberry alter table extremely_skewed set (parallel_workers = 2); @@ -85,6 +94,8 @@ update pg_class where relname = 'extremely_skewed'; -- Make a relation with a couple of enormous tuples. create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. alter table wide set (parallel_workers = 2); ANALYZE wide; -- The "optimal" case: the hash table fits in memory; we plan for 1 @@ -319,7 +330,7 @@ $$); select count(*) from simple r full outer join simple s using (id); count ------- - 20000 + 60000 (1 row) rollback to settings; @@ -574,9 +585,13 @@ rollback to settings; -- Exercise rescans. We'll turn off parallel_leader_participation so -- that we can check that instrumentation comes back correctly. create table join_foo as select generate_series(1, 3) as id, 'xxxxx'::text as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze join_foo; alter table join_foo set (parallel_workers = 0); create table join_bar as select generate_series(1, 20000) as id, 'xxxxx'::text as t; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. analyze join_bar; alter table join_bar set (parallel_workers = 2); -- multi-batch with rescan, parallel-oblivious @@ -854,23 +869,23 @@ savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); - QUERY PLAN -------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------- Finalize Aggregate - -> Gather - Workers Planned: 2 + -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate - -> Parallel Hash Full Join + -> Hash Full Join Hash Cond: (r.id = s.id) - -> Parallel Seq Scan on simple r - -> Parallel Hash - -> Parallel Seq Scan on simple s + -> Seq Scan on simple r + -> Hash + -> Seq Scan on simple s + Optimizer: Postgres query optimizer (9 rows) select count(*) from simple r full outer join simple s using (id); count ------- - 20000 + 60000 (1 row) rollback to settings; @@ -935,23 +950,25 @@ savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); - QUERY PLAN -------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------ Finalize Aggregate - -> Gather - Workers Planned: 2 + -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate - -> Parallel Hash Full Join + -> Hash Full Join Hash Cond: ((0 - s.id) = r.id) - -> Parallel Seq Scan on simple s - -> Parallel Hash - -> Parallel Seq Scan on simple r -(9 rows) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: (0 - s.id) + -> Seq Scan on simple s + -> Hash + -> Seq Scan on simple r + Optimizer: Postgres query optimizer +(11 rows) select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); - count -------- - 40000 + count +-------- + 120000 (1 row) rollback to settings; @@ -1013,7 +1030,11 @@ rollback to settings; savepoint settings; set max_parallel_workers_per_gather = 0; create table join_hash_t_small(a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table join_hash_t_big(b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into join_hash_t_small select i%100 from generate_series(0, 3000)i; insert into join_hash_t_big select i%100000 from generate_series(1, 100000)i ; analyze join_hash_t_small; @@ -1031,17 +1052,25 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b (7 rows) rollback to settings; +rollback; -- Hash join reuses the HOT status bit to indicate match status. This can only -- be guaranteed to produce correct results if all the hash join tuple match -- bits are reset before reuse. This is done upon loading them into the -- hashtable. +begin; SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; SET enable_parallel_hash = on; SET min_parallel_table_scan_size = 0; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; CREATE TABLE hjtest_matchbits_t1(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE hjtest_matchbits_t2(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO hjtest_matchbits_t1 VALUES (1); INSERT INTO hjtest_matchbits_t2 VALUES (2); -- Update should create a HOT tuple. If this status bit isn't cleared, we won't @@ -1064,8 +1093,8 @@ SET enable_parallel_hash = off; SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; id | id ----+---- - 1 | | 2 + 1 | (2 rows) ROLLBACK TO settings; @@ -1085,7 +1114,11 @@ BEGIN; SET LOCAL enable_sort = OFF; -- avoid mergejoins SET LOCAL from_collapse_limit = 1; -- allows easy changing of join order CREATE TABLE hjtest_1 (a text, b int, id int, c bool); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE TABLE hjtest_2 (a bool, id int, b text, c int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 2, 1, false); -- matches INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 1, 2, false); -- fails id join condition INSERT INTO hjtest_1(a, b, id, c) VALUES ('text', 20, 1, false); -- fails < 50 @@ -1142,8 +1175,8 @@ WHERE SubPlan 2 -> Result Output: (hjtest_1.b * 5) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'off' Optimizer: Postgres query optimizer - Settings: enable_sort=off, from_collapse_limit=1 (38 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 @@ -1206,8 +1239,8 @@ WHERE SubPlan 3 -> Result Output: (hjtest_2.c * 5) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'off' Optimizer: Postgres query optimizer - Settings: enable_sort=off, from_collapse_limit=1 (38 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 diff --git a/src/test/regress/expected/join_hash_optimizer.out b/src/test/regress/expected/join_hash_optimizer.out index 053d0ef4898..1835bfa4f31 100644 --- a/src/test/regress/expected/join_hash_optimizer.out +++ b/src/test/regress/expected/join_hash_optimizer.out @@ -10,6 +10,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or -- buckets) will be required because it can vary, but we can in some @@ -115,7 +118,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -156,7 +159,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -197,7 +200,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -241,7 +244,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -283,7 +286,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -325,7 +328,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: s.id -> Seq Scan on simple s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select count(*) from simple r join simple s using (id); @@ -344,6 +347,13 @@ $$); t | f (1 row) +-- parallel full multi-batch hash join +select count(*) from simple r full outer join simple s using (id); + count +------- + 60000 +(1 row) + rollback to settings; -- The "bad" case: during execution we need to increase number of -- batches; in this case we plan for 1 batch, and increase at least a @@ -356,8 +366,8 @@ set local work_mem = '128kB'; set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_mem explain (costs off) select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -367,8 +377,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); count @@ -395,8 +405,8 @@ set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_m set local enable_parallel_hash = off; explain (costs off) select count(*) from simple r join bigger_than_it_looks s using (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -406,8 +416,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) from simple r join bigger_than_it_looks s using (id); count @@ -434,8 +444,8 @@ set local statement_mem = '1000kB'; -- GPDB uses statement_mem instead of work_m set local enable_parallel_hash = on; explain (costs off) select count(*) from simple r join bigger_than_it_looks s using (id); - QUERY PLAN ------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------- Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -445,8 +455,8 @@ explain (costs off) -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on bigger_than_it_looks s - Optimizer: Pivotal Optimizer (GPORCA) -(13 rows) + Optimizer: GPORCA +(10 rows) select count(*) from simple r join bigger_than_it_looks s using (id); count @@ -490,7 +500,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -534,7 +544,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -578,7 +588,7 @@ HINT: For non-partitioned tables, run analyze (). For -> Hash -> Broadcast Motion 3:3 (slice2; segments: 3) -> Seq Scan on extremely_skewed s - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (10 rows) select count(*) from simple r join extremely_skewed s using (id); @@ -643,8 +653,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -662,7 +672,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -701,8 +711,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -720,7 +730,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -760,8 +770,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -779,7 +789,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -818,8 +828,8 @@ explain (costs off) select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Finalize Aggregate -> Gather Motion 3:1 (slice1; segments: 3) -> Partial Aggregate @@ -837,7 +847,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice4; segments: 3) Hash Key: b2.id -> Seq Scan on join_bar b2 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (18 rows) select count(*) from join_foo @@ -891,8 +901,9 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join savepoint settings; +set enable_parallel_hash = off; set local max_parallel_workers_per_gather = 2; explain (costs off) select count(*) from simple r full outer join simple s using (id); @@ -920,7 +931,36 @@ select count(*) from simple r full outer join simple s using (id); (1 row) rollback to settings; --- An full outer join where every record is not matched. +-- parallelism is possible with parallel-aware full hash join +savepoint settings; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s using (id); + QUERY PLAN +------------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Hash Full Join + Hash Cond: (r.id = s.id) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: r.id + -> Seq Scan on simple r + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: s.id + -> Seq Scan on simple s + Optimizer: GPORCA +(13 rows) + +select count(*) from simple r full outer join simple s using (id); + count +------- + 60000 +(1 row) + +rollback to settings; +-- A full outer join where every record is not matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; @@ -950,7 +990,37 @@ select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); (1 row) rollback to settings; --- parallelism not possible with parallel-oblivious outer hash join +-- parallelism not possible with parallel-oblivious full hash join +savepoint settings; +set enable_parallel_hash = off; +set local max_parallel_workers_per_gather = 2; +explain (costs off) + select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + QUERY PLAN +------------------------------------------------------------------------------ + Finalize Aggregate + -> Gather Motion 3:1 (slice1; segments: 3) + -> Partial Aggregate + -> Hash Full Join + Hash Cond: (r.id = (0 - s.id)) + -> Redistribute Motion 3:3 (slice2; segments: 3) + Hash Key: r.id + -> Seq Scan on simple r + -> Hash + -> Redistribute Motion 3:3 (slice3; segments: 3) + Hash Key: (0 - s.id) + -> Seq Scan on simple s + Optimizer: GPORCA +(13 rows) + +select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); + count +-------- + 120000 +(1 row) + +rollback to settings; +-- parallelism is possible with parallel-aware full hash join savepoint settings; set local max_parallel_workers_per_gather = 2; explain (costs off) @@ -1012,7 +1082,7 @@ explain (costs off) -> Redistribute Motion 3:3 (slice3; segments: 3) Hash Key: wide_1.id -> Seq Scan on wide wide_1 - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (13 rows) select length(max(s.t)) @@ -1060,11 +1130,57 @@ explain (costs off) select * from join_hash_t_small, join_hash_t_big where a = b -> Seq Scan on join_hash_t_big -> Hash -> Seq Scan on join_hash_t_small - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (7 rows) rollback to settings; rollback; +-- Hash join reuses the HOT status bit to indicate match status. This can only +-- be guaranteed to produce correct results if all the hash join tuple match +-- bits are reset before reuse. This is done upon loading them into the +-- hashtable. +begin; +SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; +SET enable_parallel_hash = on; +SET min_parallel_table_scan_size = 0; +SET parallel_setup_cost = 0; +SET parallel_tuple_cost = 0; +CREATE TABLE hjtest_matchbits_t1(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE hjtest_matchbits_t2(id int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO hjtest_matchbits_t1 VALUES (1); +INSERT INTO hjtest_matchbits_t2 VALUES (2); +-- Update should create a HOT tuple. If this status bit isn't cleared, we won't +-- correctly emit the NULL-extended unmatching tuple in full hash join. +UPDATE hjtest_matchbits_t2 set id = 2; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id + ORDER BY t1.id; + id | id +----+---- + 1 | + | 2 +(2 rows) + +-- Test serial full hash join. +-- Resetting parallel_setup_cost should force a serial plan. +-- Just to be safe, however, set enable_parallel_hash to off, as parallel full +-- hash joins are only supported with shared hashtables. +RESET parallel_setup_cost; +SET enable_parallel_hash = off; +SELECT * FROM hjtest_matchbits_t1 t1 FULL JOIN hjtest_matchbits_t2 t2 ON t1.id = t2.id; + id | id +----+---- + | 2 + 1 | +(2 rows) + +ROLLBACK TO settings; +rollback; -- Verify that hash key expressions reference the correct -- nodes. Hashjoin's hashkeys need to reference its outer plan, Hash's -- need to reference Hash's outer plan (which is below HashJoin's @@ -1154,9 +1270,9 @@ WHERE Filter: (((hjtest_1.b * 5)) < 50) -> Result Output: (hjtest_1.b * 5) - Settings: enable_sort = 'off', from_collapse_limit = '1' - Optimizer: Pivotal Optimizer (GPORCA) -(49 rows) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'on' + Optimizer: GPORCA +(51 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 FROM hjtest_1, hjtest_2 @@ -1231,9 +1347,9 @@ WHERE Filter: (((hjtest_1.b * 5)) < 50) -> Result Output: (hjtest_1.b * 5) - Settings: enable_sort = 'off', from_collapse_limit = '1' - Optimizer: Pivotal Optimizer (GPORCA) -(49 rows) + Settings: enable_parallel = 'on', enable_sort = 'off', from_collapse_limit = '1', optimizer = 'on' + Optimizer: GPORCA +(51 rows) SELECT hjtest_1.a a1, hjtest_2.a a2,hjtest_1.tableoid::regclass t1, hjtest_2.tableoid::regclass t2 FROM hjtest_2, hjtest_1 diff --git a/src/test/regress/sql/cbdb_parallel.sql b/src/test/regress/sql/cbdb_parallel.sql index f9d01dd8a00..08e7aa198f9 100644 --- a/src/test/regress/sql/cbdb_parallel.sql +++ b/src/test/regress/sql/cbdb_parallel.sql @@ -1149,6 +1149,56 @@ reset gp_cte_sharing; reset enable_parallel; reset min_parallel_table_scan_size; +-- +-- Parallel Hash Full/Right Join +-- +begin; +create table pj_t1(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t2(id int, v int) with(parallel_workers=2) distributed by (id); +create table pj_t3(id int, v int) with(parallel_workers=0) distributed by (id); + +-- pj_t1 is 3x larger than pj_t2 so the planner hashes the smaller pj_t2 +-- and probes with pj_t1, producing a genuine Parallel Hash Right Join plan. +insert into pj_t1 select i, i from generate_series(1,30000)i; +insert into pj_t2 select i, i from generate_series(25001,35000)i; +insert into pj_t3 select i, i from generate_series(1,10000)i; +analyze pj_t1; +analyze pj_t2; +analyze pj_t3; + +set local enable_parallel = on; +set local min_parallel_table_scan_size = 0; + +-- 12_P_12_10: Parallel Hash Full Join: HashedWorkers FULL JOIN HashedWorkers -> HashedOJ(parallel) +explain(costs off, locus) +select count(*) from pj_t1 full join pj_t2 using (id); +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 full join pj_t2 using (id); +set local enable_parallel = on; +select count(*) from pj_t1 full join pj_t2 using (id); + +-- Parallel Hash Right Join: pj_t1 (30K) is larger, so the planner hashes the smaller pj_t2 +-- (10K) as the build side and probes with pj_t1; result locus HashedWorkers(parallel) +explain(costs off, locus) +select count(*) from pj_t1 right join pj_t2 using (id); +-- correctness: parallel result matches non-parallel +set local enable_parallel = off; +select count(*) from pj_t1 right join pj_t2 using (id); +set local enable_parallel = on; +select count(*) from pj_t1 right join pj_t2 using (id); + +-- Locus propagation: HashedOJ(parallel) followed by INNER JOIN with Hashed(serial) +-- The full join result (HashedOJ,parallel=2) is joined with pj_t3 (Hashed,serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj inner join pj_t3 using (id); + +-- Locus propagation: HashedOJ(parallel) followed by FULL JOIN with Hashed(serial) +explain(costs off, locus) +select count(*) from (pj_t1 full join pj_t2 using (id)) fj full join pj_t3 using (id); + +abort; + -- start_ignore drop schema test_parallel cascade; -- end_ignore diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 0115489a6b9..2978e155ecd 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -13,6 +13,9 @@ set allow_system_table_mods=on; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; +-- CBDB: disable CBDB parallel for these PG-originated tests; parallel full join +-- is tested separately in cbdb_parallel.sql. +set local enable_parallel = off; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or @@ -543,7 +546,10 @@ rollback; -- be guaranteed to produce correct results if all the hash join tuple match -- bits are reset before reuse. This is done upon loading them into the -- hashtable. +begin; SAVEPOINT settings; +-- CBDB: disable CBDB parallel; the serial full join match-bit test is what matters here. +SET enable_parallel = off; SET enable_parallel_hash = on; SET min_parallel_table_scan_size = 0; SET parallel_setup_cost = 0; From 468967b8ca9a728f28408bd7938630125101e2c2 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:54:48 +0300 Subject: [PATCH 054/167] Widen MotionLayerState stat counters from uint32 to uint64 (#1647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MotionLayerState accumulates stats across all MotionNodeEntry instances. Per-node entries already use uint64. The global sum ≥ any individual node, so it overflows first — at 4GB. Fix by widening to uint64. Also fix the debug elog() format specifiers to match. --- src/backend/cdb/motion/cdbmotion.c | 4 ++-- src/include/cdb/cdbinterconnect.h | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/cdb/motion/cdbmotion.c b/src/backend/cdb/motion/cdbmotion.c index 24b112cd1bd..ad3b1c49a65 100644 --- a/src/backend/cdb/motion/cdbmotion.c +++ b/src/backend/cdb/motion/cdbmotion.c @@ -133,8 +133,8 @@ RemoveMotionLayer(MotionLayerState *mlStates) /* Emit statistics to log */ if (gp_log_interconnect >= GPVARS_VERBOSITY_VERBOSE) elog(LOG, "RemoveMotionLayer(): dumping stats\n" - " Sent: %9u chunks %9u total bytes %9u tuple bytes\n" - " Received: %9u chunks %9u total bytes %9u tuple bytes; " + " Sent: %9" INT64_MODIFIER "u chunks %9" INT64_MODIFIER "u total bytes %9" INT64_MODIFIER "u tuple bytes\n" + " Received: %9" INT64_MODIFIER "u chunks %9" INT64_MODIFIER "u total bytes %9" INT64_MODIFIER "u tuple bytes; " "%9u chunkproc calls\n", mlStates->stat_total_chunks_sent, mlStates->stat_total_bytes_sent, diff --git a/src/include/cdb/cdbinterconnect.h b/src/include/cdb/cdbinterconnect.h index 5204d4c1b94..c6c64de9590 100644 --- a/src/include/cdb/cdbinterconnect.h +++ b/src/include/cdb/cdbinterconnect.h @@ -154,13 +154,13 @@ typedef struct MotionLayerState /* * GLOBAL MOTION-LAYER STATISTICS */ - uint32 stat_total_chunks_sent; /* Tuple-chunks sent. */ - uint32 stat_total_bytes_sent; /* Bytes sent, including headers. */ - uint32 stat_tuple_bytes_sent; /* Bytes of pure tuple-data sent. */ + uint64 stat_total_chunks_sent; /* Tuple-chunks sent. */ + uint64 stat_total_bytes_sent; /* Bytes sent, including headers. */ + uint64 stat_tuple_bytes_sent; /* Bytes of pure tuple-data sent. */ - uint32 stat_total_chunks_recvd;/* Tuple-chunks received. */ - uint32 stat_total_bytes_recvd; /* Bytes received, including headers. */ - uint32 stat_tuple_bytes_recvd; /* Bytes of pure tuple-data received. */ + uint64 stat_total_chunks_recvd;/* Tuple-chunks received. */ + uint64 stat_total_bytes_recvd; /* Bytes received, including headers. */ + uint64 stat_tuple_bytes_recvd; /* Bytes of pure tuple-data received. */ uint32 stat_total_chunkproc_calls; /* Calls to processIncomingChunks() */ From 0ced84bbb3bb0683a4926006feb01fc308cd5f10 Mon Sep 17 00:00:00 2001 From: fairyfar Date: Sun, 29 Mar 2026 09:18:20 +0800 Subject: [PATCH 055/167] Fix the issue of duplicate counting of num_executed in gp_toolkit.gp_resgroup_status This is a defect of the original GPDB. When enabling resource group management and the transaction switches from "Assign" to "Bypass" state, the "num_executed" counter is repeatedly counted. --- src/backend/utils/resgroup/resgroup.c | 1 - .../expected/resgroup/resgroup_bypass.out | 18 ++++++++++++++++++ .../sql/resgroup/resgroup_bypass.sql | 9 +++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/backend/utils/resgroup/resgroup.c b/src/backend/utils/resgroup/resgroup.c index e474e106490..88f9acea9a0 100644 --- a/src/backend/utils/resgroup/resgroup.c +++ b/src/backend/utils/resgroup/resgroup.c @@ -3700,7 +3700,6 @@ check_and_unassign_from_resgroup(PlannedStmt* stmt) } while (!groupIncBypassedRef(&groupInfo)); bypassedGroup = groupInfo.group; - bypassedGroup->totalExecuted++; pgstat_report_resgroup(bypassedGroup->groupId); bypassedSlot.group = groupInfo.group; bypassedSlot.groupId = groupInfo.groupId; diff --git a/src/test/isolation2/expected/resgroup/resgroup_bypass.out b/src/test/isolation2/expected/resgroup/resgroup_bypass.out index 5cff41d745f..878c759c306 100644 --- a/src/test/isolation2/expected/resgroup/resgroup_bypass.out +++ b/src/test/isolation2/expected/resgroup/resgroup_bypass.out @@ -276,6 +276,24 @@ SELECT gp_inject_fault('func_init_plan_end', 'reset', 1); 1q: ... 2q: ... +-- verify the increment of num_executed in gp_toolkit.gp_resgroup_status +1: SET ROLE role_bypass; +SET +1: SELECT num_executed INTO temporary temp_num1 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +SELECT 1 +1: SELECT num_executed INTO temporary temp_num2 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +SELECT 1 +1: SELECT temp_num2.num_executed - temp_num1.num_executed AS delta FROM temp_num1, temp_num2; + delta +------- + 1 +(1 row) +1: DROP TABLE temp_num1; +DROP +1: DROP TABLE temp_num2; +DROP +1q: ... + -- cleanup -- start_ignore DROP TABLE t_bypass; diff --git a/src/test/isolation2/sql/resgroup/resgroup_bypass.sql b/src/test/isolation2/sql/resgroup/resgroup_bypass.sql index 19c50771f75..01d9d60cbd0 100644 --- a/src/test/isolation2/sql/resgroup/resgroup_bypass.sql +++ b/src/test/isolation2/sql/resgroup/resgroup_bypass.sql @@ -133,6 +133,15 @@ SELECT gp_inject_fault('func_init_plan_end', 'reset', 1); 1q: 2q: +-- verify the increment of num_executed in gp_toolkit.gp_resgroup_status +1: SET ROLE role_bypass; +1: SELECT num_executed INTO temporary temp_num1 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +1: SELECT num_executed INTO temporary temp_num2 FROM gp_toolkit.gp_resgroup_status WHERE groupname='rg_bypass'; +1: SELECT temp_num2.num_executed - temp_num1.num_executed AS delta FROM temp_num1, temp_num2; +1: DROP TABLE temp_num1; +1: DROP TABLE temp_num2; +1q: + -- cleanup -- start_ignore DROP TABLE t_bypass; From 2d99cf230b528e1b2850d55e418f716bf2d35cdb Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Wed, 1 Apr 2026 05:43:21 -0700 Subject: [PATCH 056/167] Fix ORCA choosing wrong column type for CTAS with UNION ALL (#1431) (#1645) * Fix ORCA choosing wrong column type for CTAS with UNION ALL (#1431) When removing redundant Result nodes in the ORCA post-processing, push_down_expr_mutator replaces parent Var nodes with child expressions. It already propagates typmod for Const child expressions, but missed the case where the child expression is also a Var. This caused the correctly resolved common typmod (e.g. -1 for varchar without length) to be overwritten by the child's original typmod (e.g. varchar(1)), resulting in wrong column types in the created table. * Add regression test for CTAS with UNION ALL typmod fix (#1431) Verify that ORCA produces the correct column type (character varying without length limit) when creating a table from UNION ALL of branches with different varchar lengths. --------- Co-authored-by: reshke --- src/backend/optimizer/plan/orca.c | 4 + src/test/regress/expected/union_gp.out | 30 ++++ .../regress/expected/union_gp_optimizer.out | 134 ++++++++++++------ src/test/regress/sql/union_gp.sql | 25 +++- 4 files changed, 147 insertions(+), 46 deletions(-) diff --git a/src/backend/optimizer/plan/orca.c b/src/backend/optimizer/plan/orca.c index 97f63f7a334..514385cc2e9 100644 --- a/src/backend/optimizer/plan/orca.c +++ b/src/backend/optimizer/plan/orca.c @@ -545,6 +545,10 @@ push_down_expr_mutator(Node *node, List *child_tlist) { ((Const *) child_tle->expr)->consttypmod = ((Var *) node)->vartypmod; } + else if (IsA(child_tle->expr, Var)) + { + ((Var *) child_tle->expr)->vartypmod = ((Var *) node)->vartypmod; + } return (Node *) child_tle->expr; } diff --git a/src/test/regress/expected/union_gp.out b/src/test/regress/expected/union_gp.out index 5bdae3e887c..d134f223502 100644 --- a/src/test/regress/expected/union_gp.out +++ b/src/test/regress/expected/union_gp.out @@ -2342,6 +2342,36 @@ with result as (update r_1240 set a = a +1 where a < 5 returning *) select * fro drop table r_1240; drop table p1_1240; -- +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. +-- +create table union_ctas_t1(id int, name varchar(1)); +create table union_ctas_t2(id int, name varchar(2)); +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; + atttypmod +----------- + -1 +(1 row) + +-- data should not be truncated +select * from union_ctas_result order by name; + id | name +----+------ + 1 | a + 1 | ab +(2 rows) + +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; +-- -- Clean up -- DROP TABLE IF EXISTS T_a1 CASCADE; diff --git a/src/test/regress/expected/union_gp_optimizer.out b/src/test/regress/expected/union_gp_optimizer.out index 8ff8655591d..8704f0fe7a7 100644 --- a/src/test/regress/expected/union_gp_optimizer.out +++ b/src/test/regress/expected/union_gp_optimizer.out @@ -1,7 +1,7 @@ -- Additional GPDB-added tests for UNION SET optimizer_trace_fallback=on; create temp table t_union1 (a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. select distinct a, null::integer as c from t_union1 union select a, b from t_union1; a | c @@ -44,8 +44,8 @@ LINE 1: select 1 intersect (select 1, 2 union all select 3, 4); select 1 a, row_number() over (partition by 'a') union all (select 1 a , 2 b); a | row_number ---+------------ - 1 | 2 1 | 1 + 1 | 2 (2 rows) -- This should preserve domain types @@ -104,8 +104,7 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support (1 row) CREATE TABLE union_ctas (a, b) AS SELECT 1, 2 UNION SELECT 1, 1 UNION SELECT 1, 1; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. SELECT * FROM union_ctas; a | b ---+--- @@ -116,11 +115,9 @@ SELECT * FROM union_ctas; DROP TABLE union_ctas; -- MPP-21075: push quals below union CREATE TABLE union_quals1 (a, b) AS SELECT i, i%2 from generate_series(1,10) i; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. CREATE TABLE union_quals2 (a, b) AS SELECT i%2, i from generate_series(1,10) i; -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. SELECT * FROM (SELECT a, b from union_quals1 UNION SELECT b, a from union_quals2) as foo(a,b) where a > b order by a; a | b ----+--- @@ -225,7 +222,7 @@ select distinct a from (select distinct 'A' from (select 'C' from (select disti -- on a single QE. -- CREATE TABLE test1 (id int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into test1 values (1); CREATE EXTERNAL WEB TABLE test2 (id int) EXECUTE 'echo 2' ON COORDINATOR FORMAT 'csv'; @@ -234,8 +231,8 @@ union (SELECT 'test2' as branch, id FROM test2); branch | id --------+---- - test1 | 1 test2 | 2 + test1 | 1 (2 rows) explain (SELECT 'test1' as branch, id FROM test1 LIMIT 1) @@ -243,10 +240,10 @@ union (SELECT 'test2' as branch, id FROM test2); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------ - Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..984.78 rows=1125 width=12) - -> HashAggregate (cost=0.00..984.73 rows=375 width=12) + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..985.86 rows=1125 width=12) + -> HashAggregate (cost=0.00..985.81 rows=375 width=12) Group Key: ('test1'::text), test1.id - -> Append (cost=0.00..984.65 rows=334 width=12) + -> Append (cost=0.00..985.73 rows=334 width=12) -> Redistribute Motion 1:3 (slice2) (cost=0.00..431.00 rows=1 width=12) Hash Key: ('test1'::text), test1.id -> GroupAggregate (cost=0.00..431.00 rows=1 width=12) @@ -257,16 +254,16 @@ union -> Result (cost=0.00..431.00 rows=1 width=12) -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) -> Seq Scan on test1 (cost=0.00..431.00 rows=1 width=4) - -> HashAggregate (cost=0.00..553.64 rows=334 width=12) - Group Key: ('test2'::text), test2.id - -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..553.56 rows=334 width=12) - Hash Key: ('test2'::text), test2.id - -> Streaming HashAggregate (cost=0.00..553.55 rows=334 width=12) - Group Key: 'test2'::text, test2.id - -> Result (cost=0.00..471.53 rows=333334 width=12) - -> Redistribute Motion 1:3 (slice5) (cost=0.00..467.53 rows=333334 width=4) - -> Foreign Scan on test2 (cost=0.00..449.70 rows=1000000 width=4) - Optimizer: Pivotal Optimizer (GPORCA) + -> HashAggregate (cost=0.00..554.73 rows=334 width=12) + Group Key: ('test2'::text), id + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..554.64 rows=334 width=12) + Hash Key: ('test2'::text), id + -> Streaming HashAggregate (cost=0.00..554.63 rows=334 width=12) + Group Key: 'test2'::text, id + -> Result (cost=0.00..473.73 rows=333334 width=12) + -> Redistribute Motion 1:3 (slice5) (cost=0.00..469.73 rows=333334 width=4) + -> Foreign Scan on test2 (cost=0.00..451.90 rows=1000000 width=4) + Optimizer: GPORCA (24 rows) -- @@ -320,8 +317,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN --------------------------------------------------------------------------------------- - Gather Motion 1:1 (slice1; segments: 1) (cost=1922.00..1922.00 rows=172200 width=8) - -> Append (cost=0.00..1922.00 rows=172200 width=8) + Gather Motion 1:1 (slice1; segments: 1) (cost=2783.00..2783.00 rows=172200 width=8) + -> Append (cost=0.00..2783.00 rows=172200 width=8) -> Seq Scan on rep2 (cost=0.00..961.00 rows=86100 width=8) -> Seq Scan on rep3 (cost=0.00..961.00 rows=86100 width=8) Optimizer: Postgres query optimizer @@ -353,7 +350,7 @@ INSERT INTO T_a1 SELECT i, i%5 from generate_series(1,10) i; CREATE TABLE T_b2 (b1 int, b2 int) DISTRIBUTED BY(b2); INSERT INTO T_b2 SELECT i, i%5 from generate_series(1,20) i; CREATE TABLE T_random (c1 int, c2 int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'c1' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO T_random SELECT i, i%5 from generate_series(1,30) i; --start_ignore @@ -2079,14 +2076,18 @@ insert into t1_ncols values (1, 11, 'one', '2001-01-01'); insert into t2_ncols values (2, 22, 'two', '2002-02-02'); insert into t2_ncols values (4, 44, 'four','2004-04-04'); select b from t1_ncols union all select a from t2_ncols; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. b ---- - 4 - 2 11 + 2 + 4 (3 rows) select a+100, b, d from t1_ncols union select b, a+200, d from t2_ncols order by 1; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. ?column? | b | d ----------+-----+------------ 22 | 202 | 02-02-2002 @@ -2095,15 +2096,19 @@ select a+100, b, d from t1_ncols union select b, a+200, d from t2_ncols order by (3 rows) select c, a from v1_ncols; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. c | a ------+--- one | 1 - four | 4 two | 2 + four | 4 (3 rows) with cte1(aa, b, c, d) as (select a*100, b, c, d from t1_ncols union select * from t2_ncols) select x.aa/100 aaa, x.c, y.c from cte1 x join cte1 y on x.aa=y.aa; +NOTICE: One or more columns in the following table(s) do not have statistics: t2_ncols +HINT: For non-partitioned tables, run analyze (). For partitioned tables, run analyze rootpartition (). See log for columns missing statistics. aaa | c | c -----+------+------ 0 | two | two @@ -2122,13 +2127,13 @@ NOTICE: schema "union_schema" does not exist, skipping -- end_ignore create schema union_schema; create table union_schema.t1(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table union_schema.t2(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. create table union_schema.t3(a int, b int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. set allow_system_table_mods = on; update gp_distribution_policy set numsegments = 1 @@ -2188,8 +2193,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN ----------------------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=1.23..1472.30 rows=86130 width=8) - -> Append (cost=1.23..323.90 rows=28710 width=8) + Gather Motion 3:1 (slice1; segments: 3) (cost=1.23..1615.85 rows=86130 width=8) + -> Append (cost=1.23..467.45 rows=28710 width=8) -> Hash Join (cost=1.23..2.80 rows=10 width=8) Hash Cond: (t2.b = t1.a) -> Redistribute Motion 2:3 (slice2; segments: 2) (cost=0.00..1.40 rows=20 width=4) @@ -2208,6 +2213,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data a | b | a | b ----+----+----+---- + 1 | 1 | 1 | 1 + 5 | 5 | 5 | 5 2 | 2 | 2 | 2 3 | 3 | 3 | 3 4 | 4 | 4 | 4 @@ -2216,8 +2223,6 @@ DETAIL: Unknown error: Partially Distributed Data 8 | 8 | 8 | 8 9 | 9 | 9 | 9 10 | 10 | 10 | 10 - 1 | 1 | 1 | 1 - 5 | 5 | 5 | 5 (10 rows) select union_schema.t1.a, union_schema.t2.b @@ -2229,6 +2234,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data a | b ----+---- + 1 | 1 + 5 | 5 2 | 2 3 | 3 4 | 4 @@ -2237,8 +2244,6 @@ DETAIL: Unknown error: Partially Distributed Data 8 | 8 9 | 9 10 | 10 - 1 | 1 - 5 | 5 (10 rows) truncate union_schema.t1, union_schema.t2; @@ -2276,8 +2281,8 @@ INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner DETAIL: Unknown error: Partially Distributed Data QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=1.32..1472.20 rows=86130 width=8) - -> Append (cost=1.32..323.80 rows=28710 width=8) + Gather Motion 3:1 (slice1; segments: 3) (cost=1.32..1615.75 rows=86130 width=8) + -> Append (cost=1.32..467.35 rows=28710 width=8) -> Hash Join (cost=1.32..2.70 rows=10 width=8) Hash Cond: (t1.a = t2.b) -> Seq Scan on t1 (cost=0.00..1.20 rows=20 width=4) @@ -2340,6 +2345,8 @@ reset allow_system_table_mods; create table rep (a int) distributed replicated; insert into rep select i from generate_series (1, 10) i; create table dist (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into dist select i from generate_series (1, 1000) i; analyze dist; analyze rep; @@ -2352,7 +2359,7 @@ explain select a from rep union all select a from dist; Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.03 rows=1010 width=4) -> Append (cost=0.00..862.01 rows=337 width=4) -> Result (cost=0.00..431.00 rows=4 width=4) - One-Time Filter: (gp_execution_segment() = 2) + One-Time Filter: (gp_execution_segment() = 0) -> Seq Scan on rep (cost=0.00..431.00 rows=10 width=4) -> Seq Scan on dist (cost=0.00..431.01 rows=334 width=4) Optimizer: GPORCA @@ -2368,12 +2375,12 @@ analyze rand; explain select i from generate_series(1,1000) i union all select a from rand; QUERY PLAN ---------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.28 rows=11000 width=4) + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.29 rows=11000 width=4) -> Append (cost=0.00..431.12 rows=3667 width=4) -> Result (cost=0.00..0.01 rows=334 width=4) - One-Time Filter: (gp_execution_segment() = 2) + One-Time Filter: (gp_execution_segment() = 0) -> Function Scan on generate_series (cost=0.00..0.00 rows=334 width=4) - -> Seq Scan on rand (cost=0.00..431.06 rows=3334 width=4) + -> Seq Scan on rand (cost=0.00..431.07 rows=3334 width=4) Optimizer: GPORCA (7 rows) @@ -2460,7 +2467,7 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support -> Gather Motion 3:1 (slice2; segments: 3) -> Subquery Scan on "*SELECT* 2" -> Seq Scan on p1_1240 - Optimizer: Postgres-based planner + Optimizer: Postgres query optimizer (11 rows) with result as (update r_1240 set a = a +1 where a < 5 returning *) select * from result except select * from p1_1240; @@ -2475,6 +2482,43 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support drop table r_1240; drop table p1_1240; -- +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. +-- +create table union_ctas_t1(id int, name varchar(1)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create table union_ctas_t2(id int, name varchar(2)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'id' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; +INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner +DETAIL: Falling back to Postgres-based planner because GPORCA does not support the following feature: Queries on master-only tables + atttypmod +----------- + -1 +(1 row) + +-- data should not be truncated +select * from union_ctas_result order by name; + id | name +----+------ + 1 | a + 1 | ab +(2 rows) + +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; +-- -- Clean up -- DROP TABLE IF EXISTS T_a1 CASCADE; diff --git a/src/test/regress/sql/union_gp.sql b/src/test/regress/sql/union_gp.sql index e7cac952704..9e9e5c3a815 100644 --- a/src/test/regress/sql/union_gp.sql +++ b/src/test/regress/sql/union_gp.sql @@ -721,9 +721,32 @@ drop table r_1240; drop table p1_1240; -- --- Clean up +-- Test CTAS with UNION ALL when branches have different typmods (issue #1431). +-- ORCA should resolve the output column type to character varying (no length), +-- same as the Postgres planner, instead of picking the first branch's typmod. -- +create table union_ctas_t1(id int, name varchar(1)); +create table union_ctas_t2(id int, name varchar(2)); +insert into union_ctas_t1 values (1, 'a'); +insert into union_ctas_t2 values (1, 'ab'); + +create table union_ctas_result as + (select id, name from union_ctas_t1) + union all + (select id, name from union_ctas_t2); + +-- name column should be "character varying" without length, not varchar(1) +select atttypmod from pg_attribute +where attrelid = 'union_ctas_result'::regclass and attname = 'name'; + +-- data should not be truncated +select * from union_ctas_result order by name; +drop table union_ctas_t1, union_ctas_t2, union_ctas_result; + +-- +-- Clean up +-- DROP TABLE IF EXISTS T_a1 CASCADE; DROP TABLE IF EXISTS T_b2 CASCADE; DROP TABLE IF EXISTS T_random CASCADE; From 2703505569f9ef4e793c4e7262c09a9e057c8ca9 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 1 Apr 2026 20:48:48 +0800 Subject: [PATCH 057/167] Revert "Fix COPY TO returning 0 rows during concurrent reorganize" This reverts commit f97979911465650b6626eeffa49af9429a11d2c6. --- .../pax/copy_to_concurrent_reorganize.out | 289 ------ .../src/test/isolation2/isolation2_schedule | 1 - .../sql/pax/copy_to_concurrent_reorganize.sql | 170 ---- src/backend/commands/copy.c | 81 -- src/backend/commands/copyto.c | 37 - .../copy_to_concurrent_reorganize.out | 918 ------------------ src/test/isolation2/isolation2_schedule | 1 - .../sql/copy_to_concurrent_reorganize.sql | 561 ----------- 8 files changed, 2058 deletions(-) delete mode 100644 contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out delete mode 100644 contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql delete mode 100644 src/test/isolation2/expected/copy_to_concurrent_reorganize.out delete mode 100644 src/test/isolation2/sql/copy_to_concurrent_reorganize.sql diff --git a/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out b/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out deleted file mode 100644 index b4beed7d035..00000000000 --- a/contrib/pax_storage/src/test/isolation2/expected/pax/copy_to_concurrent_reorganize.out +++ /dev/null @@ -1,289 +0,0 @@ --- Test: PAX table — relation-based COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- Same as test 2.1 in the main isolation2 suite but for PAX storage. - -CREATE TABLE copy_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_pax_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_pax_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_pax_test TO '/tmp/copy_reorg_pax_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_pax_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_pax_verify FROM '/tmp/copy_reorg_pax_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_pax_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_pax_verify; -DROP -DROP TABLE copy_reorg_pax_test; -DROP - --- ============================================================ --- Test 2.2c: PAX — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_pax_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_pax_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_pax_test) TO '/tmp/copy_query_reorg_pax_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_pax_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_pax_verify FROM '/tmp/copy_query_reorg_pax_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_pax_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_pax_verify; -DROP -DROP TABLE copy_query_reorg_pax_test; -DROP - --- ============================================================ --- Test 2.3c: PAX — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_pax (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (1) TO (501); -CREATE -CREATE TABLE copy_part_child2_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (501) TO (1001); -CREATE -INSERT INTO copy_part_parent_pax SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_pax; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_pax SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_pax TO '/tmp/copy_part_parent_pax.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_pax%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_part_pax_verify FROM '/tmp/copy_part_parent_pax.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_pax_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_pax_verify; -DROP -DROP TABLE copy_part_parent_pax; -DROP - --- ============================================================ --- Test 2.4c: PAX — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2c — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_pax_lookup (cat INT) DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_pax_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_pax_main (a INT, category INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_pax_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_pax_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_pax ON copy_rls_pax_main USING (category IN (SELECT cat from copy_rls_pax_lookup)); -CREATE - -CREATE ROLE copy_rls_pax_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_pax_testuser; -GRANT -GRANT ALL ON copy_rls_pax_main TO copy_rls_pax_testuser; -GRANT -GRANT ALL ON copy_rls_pax_lookup TO copy_rls_pax_testuser; -GRANT - -SELECT count(*) FROM copy_rls_pax_main; - count -------- - 1000 -(1 row) - -2: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_pax_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_pax_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_pax_verify (a INT, category INT) DISTRIBUTED BY (a); -CREATE -COPY copy_rls_pax_verify FROM '/tmp/copy_rls_pax_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_pax_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_pax_verify; -DROP -DROP POLICY p_rls_pax ON copy_rls_pax_main; -DROP -DROP TABLE copy_rls_pax_main; -DROP -DROP TABLE copy_rls_pax_lookup; -DROP -DROP ROLE copy_rls_pax_testuser; -DROP - --- ============================================================ --- Test 2.5c: PAX — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_pax_src (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_pax_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_pax_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_pax_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_pax_dst AS SELECT * FROM ctas_reorg_pax_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_pax_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_pax_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_pax_dst; -DROP -DROP TABLE ctas_reorg_pax_src; -DROP - --- NOTE: Test 2.6c (PAX variant of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/contrib/pax_storage/src/test/isolation2/isolation2_schedule b/contrib/pax_storage/src/test/isolation2/isolation2_schedule index fa163aa96b6..72fa06f5204 100644 --- a/contrib/pax_storage/src/test/isolation2/isolation2_schedule +++ b/contrib/pax_storage/src/test/isolation2/isolation2_schedule @@ -157,7 +157,6 @@ test: pax/vacuum_while_vacuum # test: uao/bad_buffer_on_temp_ao_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop -test: pax/copy_to_concurrent_reorganize # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql b/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql deleted file mode 100644 index 05ef25852e9..00000000000 --- a/contrib/pax_storage/src/test/isolation2/sql/pax/copy_to_concurrent_reorganize.sql +++ /dev/null @@ -1,170 +0,0 @@ --- Test: PAX table — relation-based COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- Same as test 2.1 in the main isolation2 suite but for PAX storage. - -CREATE TABLE copy_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_pax_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_pax_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_pax_test TO '/tmp/copy_reorg_pax_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_pax_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_reorg_pax_verify FROM '/tmp/copy_reorg_pax_test.csv'; -SELECT count(*) FROM copy_reorg_pax_verify; - --- Cleanup -DROP TABLE copy_reorg_pax_verify; -DROP TABLE copy_reorg_pax_test; - --- ============================================================ --- Test 2.2c: PAX — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_pax_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_pax_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_pax_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_pax_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_pax_test) TO '/tmp/copy_query_reorg_pax_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_pax_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_query_reorg_pax_verify FROM '/tmp/copy_query_reorg_pax_test.csv'; -SELECT count(*) FROM copy_query_reorg_pax_verify; - -DROP TABLE copy_query_reorg_pax_verify; -DROP TABLE copy_query_reorg_pax_test; - --- ============================================================ --- Test 2.3c: PAX — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_pax (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (1) TO (501); -CREATE TABLE copy_part_child2_pax PARTITION OF copy_part_parent_pax FOR VALUES FROM (501) TO (1001); -INSERT INTO copy_part_parent_pax SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_pax; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_pax SET WITH (reorganize=true); - -2&: COPY copy_part_parent_pax TO '/tmp/copy_part_parent_pax.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_pax%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_pax_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_part_pax_verify FROM '/tmp/copy_part_parent_pax.csv'; -SELECT count(*) FROM copy_part_pax_verify; - -DROP TABLE copy_part_pax_verify; -DROP TABLE copy_part_parent_pax; - --- ============================================================ --- Test 2.4c: PAX — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2c — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_pax_lookup (cat INT) DISTRIBUTED BY (cat); -INSERT INTO copy_rls_pax_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_pax_main (a INT, category INT) DISTRIBUTED BY (a); -INSERT INTO copy_rls_pax_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_pax_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_pax ON copy_rls_pax_main USING (category IN (SELECT cat from copy_rls_pax_lookup)); - -CREATE ROLE copy_rls_pax_testuser; -GRANT pg_write_server_files TO copy_rls_pax_testuser; -GRANT ALL ON copy_rls_pax_main TO copy_rls_pax_testuser; -GRANT ALL ON copy_rls_pax_lookup TO copy_rls_pax_testuser; - -SELECT count(*) FROM copy_rls_pax_main; - -2: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_pax_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_pax_testuser; COPY copy_rls_pax_main TO '/tmp/copy_rls_pax_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_pax_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_pax_verify (a INT, category INT) DISTRIBUTED BY (a); -COPY copy_rls_pax_verify FROM '/tmp/copy_rls_pax_main.csv'; -SELECT count(*) FROM copy_rls_pax_verify; - -DROP TABLE copy_rls_pax_verify; -DROP POLICY p_rls_pax ON copy_rls_pax_main; -DROP TABLE copy_rls_pax_main; -DROP TABLE copy_rls_pax_lookup; -DROP ROLE copy_rls_pax_testuser; - --- ============================================================ --- Test 2.5c: PAX — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_pax_src (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_pax_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_pax_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_pax_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_pax_dst AS SELECT * FROM ctas_reorg_pax_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_pax_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_pax_dst; - -DROP TABLE ctas_reorg_pax_dst; -DROP TABLE ctas_reorg_pax_src; - --- NOTE: Test 2.6c (PAX variant of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index c9d2ac4f968..4ccd3798067 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -58,7 +58,6 @@ #include "catalog/catalog.h" #include "catalog/gp_matview_aux.h" #include "catalog/namespace.h" -#include "catalog/pg_inherits.h" #include "catalog/pg_extprotocol.h" #include "cdb/cdbappendonlyam.h" #include "cdb/cdbaocsam.h" @@ -137,37 +136,6 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, { /* Open and lock the relation, using the appropriate lock type. */ rel = table_openrv(stmt->relation, lockmode); - - /* - * For COPY TO, refresh the active snapshot after acquiring the lock. - * - * The snapshot was originally pushed by PortalRunUtility() before - * DoCopy() was called, which means it was taken before we acquired - * the lock on the relation. If we had to wait for a conflicting lock - * (e.g., AccessExclusiveLock held by a concurrent ALTER TABLE ... - * SET WITH (reorganize=true)), the snapshot may predate the - * concurrent transaction's commit. After the lock is granted, scanning - * with such a stale snapshot would miss all tuples written by the - * concurrent transaction, resulting in COPY returning zero rows. - * - * This mirrors the approach used by exec_simple_query() for SELECT - * statements, which pops the parse/analyze snapshot and takes a fresh - * one in PortalStart() after locks have been acquired (see the comment - * at postgres.c:1859-1867). It is also consistent with how VACUUM and - * CLUSTER manage their own snapshots internally. - * - * In REPEATABLE READ or SERIALIZABLE mode, GetTransactionSnapshot() - * returns the same transaction-level snapshot regardless, making this - * a harmless no-op. - * - * We only do this for COPY TO (!is_from) because COPY FROM inserts - * data and does not scan existing tuples with a snapshot. - */ - if (!is_from && ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } } /* @@ -304,55 +272,6 @@ DoCopy(ParseState *pstate, const CopyStmt *stmt, errmsg("COPY FROM not supported with row-level security"), errhint("Use INSERT statements instead."))); - /* - * For partitioned table COPY TO: eagerly acquire AccessShareLock - * on all child partitions before refreshing the snapshot. - * - * When COPY is performed on a partitioned table, the parent - * relation's AccessShareLock is acquired above (via table_openrv) - * and Method A already refreshed the snapshot. However, the - * parent's AccessShareLock does NOT conflict with an - * AccessExclusiveLock held on a child partition by a concurrent - * reorganize. As a result, Method A's snapshot may still predate - * the child's reorganize commit. - * - * Child partition locks are acquired later, deep inside - * ExecutorStart() via ExecInitAppend(), by which time the snapshot - * has already been embedded in the QueryDesc via - * PushCopiedSnapshot() in BeginCopy(). Even a second snapshot - * refresh in BeginCopy() (after AcquireRewriteLocks) would not - * help, because AcquireRewriteLocks only locks the parent (child - * partitions are not in the initial range table of - * "SELECT * FROM parent"). - * - * The fix: call find_all_inheritors() with AccessShareLock to - * acquire locks on every child partition NOW, before building the - * query. If a child partition's reorganize holds - * AccessExclusiveLock, this call blocks until that transaction - * commits. Once it returns, all child-level reorganize operations - * have committed, and a fresh snapshot taken here will see all - * reorganized child data. - * - * find_all_inheritors() acquires locks that persist to end of - * transaction. The executor will re-acquire them during scan - * initialization, which is a lock-manager no-op. - */ - if (!is_from && rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) - { - List *part_oids; - - part_oids = find_all_inheritors(RelationGetRelid(rel), - AccessShareLock, NULL); - list_free(part_oids); - - /* Refresh snapshot: all child partition locks now held */ - if (ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } - } - /* * Build target list * diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 88e61305250..871a973235e 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -1198,43 +1198,6 @@ BeginCopy(ParseState *pstate, Assert(query->utilityStmt == NULL); - /* - * Refresh the active snapshot after pg_analyze_and_rewrite() has - * acquired all necessary relation locks via AcquireRewriteLocks(). - * - * The snapshot in use was pushed by PortalRunUtility() before DoCopy() - * was called -- before any table locks were acquired. If - * AcquireRewriteLocks() had to wait for a conflicting - * AccessExclusiveLock (e.g., held by a concurrent ALTER TABLE ... - * SET WITH (reorganize=true)), the lock wait is now over and the - * reorganize transaction has committed. The snapshot taken before the - * wait does not reflect that commit: after reorganize completes, - * swap_relation_files() has replaced the physical storage, so old - * tuples no longer exist and the new tuples have xmin = reorganize_xid - * which is not yet visible in the pre-wait snapshot. Scanning with - * the stale snapshot returns 0 rows -- a violation of transaction - * atomicity (the reader must see either all old rows or all new rows). - * - * By refreshing the snapshot here -- after all locks are acquired -- - * we guarantee that the query will see the committed post-reorganize - * data. - * - * This applies to: - * - Pure query-based COPY TO: COPY (SELECT ...) TO - * - RLS table COPY TO: converted to query-based in DoCopy(); the - * RLS policy references an external lookup table whose lock is - * acquired by AcquireRewriteLocks(). - * - * In REPEATABLE READ or SERIALIZABLE isolation, - * GetTransactionSnapshot() returns the same transaction-level - * snapshot, making this a harmless no-op. - */ - if (ActiveSnapshotSet()) - { - PopActiveSnapshot(); - PushActiveSnapshot(GetTransactionSnapshot()); - } - /* * Similarly the grammar doesn't enforce the presence of a RETURNING * clause, but this is required here. diff --git a/src/test/isolation2/expected/copy_to_concurrent_reorganize.out b/src/test/isolation2/expected/copy_to_concurrent_reorganize.out deleted file mode 100644 index 0a7dfd38801..00000000000 --- a/src/test/isolation2/expected/copy_to_concurrent_reorganize.out +++ /dev/null @@ -1,918 +0,0 @@ --- Test: COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- --- Tests 2.1: Core fix (relation-based COPY TO) --- Tests 2.2-2.5: Extended fixes for query-based, partitioned, RLS, and CTAS paths - --- ============================================================ --- Test 2.1: relation-based COPY TO + concurrent reorganize --- Reproduces issue #1545: COPY TO should return correct row count --- after waiting for reorganize to release AccessExclusiveLock. --- ============================================================ - -CREATE TABLE copy_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock --- At this point PortalRunUtility has already acquired a snapshot (before reorganize commits), --- then DoCopy tries to acquire the lock and blocks. -2&: COPY copy_reorg_test TO '/tmp/copy_reorg_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_verify FROM '/tmp/copy_reorg_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_verify; -DROP -DROP TABLE copy_reorg_test; -DROP - --- ============================================================ --- Test 2.2: query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after pg_analyze_and_rewrite() --- acquires all relation locks via AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_test; - count -------- - 1000 -(1 row) - --- Session 1: reorganize holds AccessExclusiveLock -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_test SET WITH (reorganize=true); -ALTER - --- Session 2: query-based COPY TO blocks (lock acquired in pg_analyze_and_rewrite -> AcquireRewriteLocks) -2&: COPY (SELECT * FROM copy_query_reorg_test) TO '/tmp/copy_query_reorg_test.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_query_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_verify FROM '/tmp/copy_query_reorg_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_query_reorg_verify; -DROP -DROP TABLE copy_query_reorg_test; -DROP - --- ============================================================ --- Test 2.3: partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to eagerly lock all child --- partitions before refreshing the snapshot, ensuring the snapshot sees all --- child reorganize commits before the query is built. --- ============================================================ - -CREATE TABLE copy_part_parent (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1 PARTITION OF copy_part_parent FOR VALUES FROM (1) TO (501); -CREATE -CREATE TABLE copy_part_child2 PARTITION OF copy_part_parent FOR VALUES FROM (501) TO (1001); -CREATE -INSERT INTO copy_part_parent SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent; - count -------- - 1000 -(1 row) - --- Session 1: reorganize the child partition -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1 SET WITH (reorganize=true); -ALTER - --- Session 2: COPY parent TO (internally converted to query-based, child lock acquired in analyze phase) -2&: COPY copy_part_parent TO '/tmp/copy_part_parent.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_part_verify (a INT, b INT) DISTRIBUTED BY (a); -CREATE -COPY copy_part_verify FROM '/tmp/copy_part_parent.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_part_verify; -DROP -DROP TABLE copy_part_parent; -DROP - --- ============================================================ --- Test 2.4: RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2 — BeginCopy() refreshes snapshot after AcquireRewriteLocks() --- which also acquires the lock on the RLS policy's lookup table. --- ============================================================ - -CREATE TABLE copy_rls_lookup (cat INT) DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_main (a INT, category INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls ON copy_rls_main USING (category IN (SELECT cat FROM copy_rls_lookup)); -CREATE - --- Create non-superuser to trigger RLS (needs pg_write_server_files to COPY TO file) -CREATE ROLE copy_rls_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_testuser; -GRANT -GRANT ALL ON copy_rls_main TO copy_rls_testuser; -GRANT -GRANT ALL ON copy_rls_lookup TO copy_rls_testuser; -GRANT - -SELECT count(*) FROM copy_rls_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; -SET 400 - --- Session 1: reorganize the lookup table -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_lookup SET WITH (reorganize=true); -ALTER - --- Session 2: COPY TO as non-superuser (RLS active, internally converted to query-based) -2&: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -SET 400 - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; -RESET - --- Verify: should match baseline count (400 rows filtered by RLS) -RESET ROLE; -RESET -CREATE TABLE copy_rls_verify (a INT, category INT) DISTRIBUTED BY (a); -CREATE -COPY copy_rls_verify FROM '/tmp/copy_rls_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_verify; - count -------- - 400 -(1 row) - --- Cleanup -DROP TABLE copy_rls_verify; -DROP -DROP POLICY p_rls ON copy_rls_main; -DROP -DROP TABLE copy_rls_main; -DROP -DROP TABLE copy_rls_lookup; -DROP -DROP ROLE copy_rls_testuser; -DROP - --- ============================================================ --- Test 2.5: CTAS + concurrent reorganize --- Fixed as a side effect: CTAS goes through pg_analyze_and_rewrite() + --- AcquireRewriteLocks(), so the snapshot refresh in BeginCopy() also fixes it. --- ============================================================ - -CREATE TABLE ctas_reorg_src (a INT, b INT) DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_src; - count -------- - 1000 -(1 row) - --- Session 1: reorganize -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_src SET WITH (reorganize=true); -ALTER - --- Session 2: CTAS should block (lock acquired in executor or analyze phase) -2&: CREATE TABLE ctas_reorg_dst AS SELECT * FROM ctas_reorg_src DISTRIBUTED BY (a); - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit -1: COMMIT; -COMMIT - --- Session 2: Complete -2<: <... completed> -CREATE 1000 - --- Verify row count after CTAS completes -SELECT count(*) FROM ctas_reorg_dst; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE ctas_reorg_dst; -DROP -DROP TABLE ctas_reorg_src; -DROP - --- NOTE: Test 2.6 (change distribution key + query-based COPY TO) removed because --- ALTER TABLE SET DISTRIBUTED BY + concurrent query-based COPY TO causes a server --- crash (pre-existing Cloudberry bug, not related to this fix). - --- ============================================================ --- Test 2.1a: AO row table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized row-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_row_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_ao_row_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_row_test TO '/tmp/copy_reorg_ao_row_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_ao_row_verify FROM '/tmp/copy_reorg_ao_row_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_ao_row_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_ao_row_verify; -DROP -DROP TABLE copy_reorg_ao_row_test; -DROP - --- ============================================================ --- Test 2.1b: AO column table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized column-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_col_test; - count -------- - 1000 -(1 row) - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -BEGIN -1: ALTER TABLE copy_reorg_ao_col_test SET WITH (reorganize=true); -ALTER - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_col_test TO '/tmp/copy_reorg_ao_col_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; -COMMIT - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: <... completed> -COPY 1000 - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_reorg_ao_col_verify FROM '/tmp/copy_reorg_ao_col_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_reorg_ao_col_verify; - count -------- - 1000 -(1 row) - --- Cleanup -DROP TABLE copy_reorg_ao_col_verify; -DROP -DROP TABLE copy_reorg_ao_col_test; -DROP - --- ============================================================ --- Test 2.2a: AO row — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_ao_row_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_ao_row_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_ao_row_test) TO '/tmp/copy_query_reorg_ao_row_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_ao_row_verify FROM '/tmp/copy_query_reorg_ao_row_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_ao_row_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_ao_row_verify; -DROP -DROP TABLE copy_query_reorg_ao_row_test; -DROP - --- ============================================================ --- Test 2.2b: AO column — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_query_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_query_reorg_ao_col_test; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_query_reorg_ao_col_test SET WITH (reorganize=true); -ALTER - -2&: COPY (SELECT * FROM copy_query_reorg_ao_col_test) TO '/tmp/copy_query_reorg_ao_col_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_query_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_query_reorg_ao_col_verify FROM '/tmp/copy_query_reorg_ao_col_test.csv'; -COPY 1000 -SELECT count(*) FROM copy_query_reorg_ao_col_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_query_reorg_ao_col_verify; -DROP -DROP TABLE copy_query_reorg_ao_col_test; -DROP - --- ============================================================ --- Test 2.3a: AO row — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_row (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (1) TO (501) USING ao_row; -CREATE -CREATE TABLE copy_part_child2_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (501) TO (1001) USING ao_row; -CREATE -INSERT INTO copy_part_parent_ao_row SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_ao_row; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_ao_row SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_ao_row TO '/tmp/copy_part_parent_ao_row.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_ao_row%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_part_ao_row_verify FROM '/tmp/copy_part_parent_ao_row.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_ao_row_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_ao_row_verify; -DROP -DROP TABLE copy_part_parent_ao_row; -DROP - --- ============================================================ --- Test 2.3b: AO column — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_col (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE -CREATE TABLE copy_part_child1_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (1) TO (501) USING ao_column; -CREATE -CREATE TABLE copy_part_child2_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (501) TO (1001) USING ao_column; -CREATE -INSERT INTO copy_part_parent_ao_col SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM copy_part_parent_ao_col; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_part_child1_ao_col SET WITH (reorganize=true); -ALTER - -2&: COPY copy_part_parent_ao_col TO '/tmp/copy_part_parent_ao_col.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'COPY copy_part_parent_ao_col%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -COPY 1000 - -CREATE TABLE copy_part_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_part_ao_col_verify FROM '/tmp/copy_part_parent_ao_col.csv'; -COPY 1000 -SELECT count(*) FROM copy_part_ao_col_verify; - count -------- - 1000 -(1 row) - -DROP TABLE copy_part_ao_col_verify; -DROP -DROP TABLE copy_part_parent_ao_col; -DROP - --- ============================================================ --- Test 2.4a: AO row — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_row_lookup (cat INT) USING ao_row DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_ao_row_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_ao_row_main (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_ao_row_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_ao_row_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_ao_row ON copy_rls_ao_row_main USING (category IN (SELECT cat FROM copy_rls_ao_row_lookup)); -CREATE - -CREATE ROLE copy_rls_ao_row_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_ao_row_testuser; -GRANT -GRANT ALL ON copy_rls_ao_row_main TO copy_rls_ao_row_testuser; -GRANT -GRANT ALL ON copy_rls_ao_row_lookup TO copy_rls_ao_row_testuser; -GRANT - -SELECT count(*) FROM copy_rls_ao_row_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_ao_row_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_ao_row_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_ao_row_verify (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -CREATE -COPY copy_rls_ao_row_verify FROM '/tmp/copy_rls_ao_row_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_ao_row_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_ao_row_verify; -DROP -DROP POLICY p_rls_ao_row ON copy_rls_ao_row_main; -DROP -DROP TABLE copy_rls_ao_row_main; -DROP -DROP TABLE copy_rls_ao_row_lookup; -DROP -DROP ROLE copy_rls_ao_row_testuser; -DROP - --- ============================================================ --- Test 2.4b: AO column — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_col_lookup (cat INT) USING ao_column DISTRIBUTED BY (cat); -CREATE -INSERT INTO copy_rls_ao_col_lookup SELECT i FROM generate_series(1, 2) i; -INSERT 2 - -CREATE TABLE copy_rls_ao_col_main (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO copy_rls_ao_col_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; -INSERT 1000 - -ALTER TABLE copy_rls_ao_col_main ENABLE ROW LEVEL SECURITY; -ALTER -CREATE POLICY p_rls_ao_col ON copy_rls_ao_col_main USING (category IN (SELECT cat FROM copy_rls_ao_col_lookup)); -CREATE - -CREATE ROLE copy_rls_ao_col_testuser; -CREATE -GRANT pg_write_server_files TO copy_rls_ao_col_testuser; -GRANT -GRANT ALL ON copy_rls_ao_col_main TO copy_rls_ao_col_testuser; -GRANT -GRANT ALL ON copy_rls_ao_col_lookup TO copy_rls_ao_col_testuser; -GRANT - -SELECT count(*) FROM copy_rls_ao_col_main; - count -------- - 1000 -(1 row) - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; -SET 400 - -1: BEGIN; -BEGIN -1: ALTER TABLE copy_rls_ao_col_lookup SET WITH (reorganize=true); -ALTER - -2&: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE '%COPY copy_rls_ao_col_main%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -SET 400 - -2: RESET ROLE; -RESET - -RESET ROLE; -RESET -CREATE TABLE copy_rls_ao_col_verify (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -CREATE -COPY copy_rls_ao_col_verify FROM '/tmp/copy_rls_ao_col_main.csv'; -COPY 400 -SELECT count(*) FROM copy_rls_ao_col_verify; - count -------- - 400 -(1 row) - -DROP TABLE copy_rls_ao_col_verify; -DROP -DROP POLICY p_rls_ao_col ON copy_rls_ao_col_main; -DROP -DROP TABLE copy_rls_ao_col_main; -DROP -DROP TABLE copy_rls_ao_col_lookup; -DROP -DROP ROLE copy_rls_ao_col_testuser; -DROP - --- ============================================================ --- Test 2.5a: AO row — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_row_src (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_ao_row_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_ao_row_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_ao_row_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_ao_row_dst AS SELECT * FROM ctas_reorg_ao_row_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_row_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_ao_row_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_ao_row_dst; -DROP -DROP TABLE ctas_reorg_ao_row_src; -DROP - --- ============================================================ --- Test 2.5b: AO column — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_col_src (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -CREATE -INSERT INTO ctas_reorg_ao_col_src SELECT i, i FROM generate_series(1, 1000) i; -INSERT 1000 - -SELECT count(*) FROM ctas_reorg_ao_col_src; - count -------- - 1000 -(1 row) - -1: BEGIN; -BEGIN -1: ALTER TABLE ctas_reorg_ao_col_src SET WITH (reorganize=true); -ALTER - -2&: CREATE TABLE ctas_reorg_ao_col_dst AS SELECT * FROM ctas_reorg_ao_col_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_col_dst%' AND wait_event_type = 'Lock'; - ?column? ----------- - t -(1 row) - -1: COMMIT; -COMMIT -2<: <... completed> -CREATE 1000 - -SELECT count(*) FROM ctas_reorg_ao_col_dst; - count -------- - 1000 -(1 row) - -DROP TABLE ctas_reorg_ao_col_dst; -DROP -DROP TABLE ctas_reorg_ao_col_src; -DROP - --- NOTE: Tests 2.6a/2.6b (AO variants of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 4a0f9dc6925..d9d33ad76e4 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -152,7 +152,6 @@ test: uao/fast_analyze_row test: uao/create_index_allows_readonly_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop -test: copy_to_concurrent_reorganize # below test(s) inject faults so each of them need to be in a separate group test: segwalrep/master_wal_switch diff --git a/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql b/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql deleted file mode 100644 index 3473193d142..00000000000 --- a/src/test/isolation2/sql/copy_to_concurrent_reorganize.sql +++ /dev/null @@ -1,561 +0,0 @@ --- Test: COPY TO concurrent with ALTER TABLE SET WITH (reorganize=true) --- Issue: https://github.com/apache/cloudberry/issues/1545 --- --- Tests 2.1: Core fix (relation-based COPY TO) --- Tests 2.2-2.5: Extended fixes for query-based, partitioned, RLS, and CTAS paths - --- ============================================================ --- Test 2.1: relation-based COPY TO + concurrent reorganize --- Reproduces issue #1545: COPY TO should return correct row count --- after waiting for reorganize to release AccessExclusiveLock. --- ============================================================ - -CREATE TABLE copy_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_reorg_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock --- At this point PortalRunUtility has already acquired a snapshot (before reorganize commits), --- then DoCopy tries to acquire the lock and blocks. -2&: COPY copy_reorg_test TO '/tmp/copy_reorg_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_reorg_verify FROM '/tmp/copy_reorg_test.csv'; -SELECT count(*) FROM copy_reorg_verify; - --- Cleanup -DROP TABLE copy_reorg_verify; -DROP TABLE copy_reorg_test; - --- ============================================================ --- Test 2.2: query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after pg_analyze_and_rewrite() --- acquires all relation locks via AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_test (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_test; - --- Session 1: reorganize holds AccessExclusiveLock -1: BEGIN; -1: ALTER TABLE copy_query_reorg_test SET WITH (reorganize=true); - --- Session 2: query-based COPY TO blocks (lock acquired in pg_analyze_and_rewrite -> AcquireRewriteLocks) -2&: COPY (SELECT * FROM copy_query_reorg_test) TO '/tmp/copy_query_reorg_test.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_query_reorg_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_query_reorg_verify FROM '/tmp/copy_query_reorg_test.csv'; -SELECT count(*) FROM copy_query_reorg_verify; - --- Cleanup -DROP TABLE copy_query_reorg_verify; -DROP TABLE copy_query_reorg_test; - --- ============================================================ --- Test 2.3: partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to eagerly lock all child --- partitions before refreshing the snapshot, ensuring the snapshot sees all --- child reorganize commits before the query is built. --- ============================================================ - -CREATE TABLE copy_part_parent (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1 PARTITION OF copy_part_parent FOR VALUES FROM (1) TO (501); -CREATE TABLE copy_part_child2 PARTITION OF copy_part_parent FOR VALUES FROM (501) TO (1001); -INSERT INTO copy_part_parent SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent; - --- Session 1: reorganize the child partition -1: BEGIN; -1: ALTER TABLE copy_part_child1 SET WITH (reorganize=true); - --- Session 2: COPY parent TO (internally converted to query-based, child lock acquired in analyze phase) -2&: COPY copy_part_parent TO '/tmp/copy_part_parent.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_part_verify (a INT, b INT) DISTRIBUTED BY (a); -COPY copy_part_verify FROM '/tmp/copy_part_parent.csv'; -SELECT count(*) FROM copy_part_verify; - --- Cleanup -DROP TABLE copy_part_verify; -DROP TABLE copy_part_parent; - --- ============================================================ --- Test 2.4: RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.2 — BeginCopy() refreshes snapshot after AcquireRewriteLocks() --- which also acquires the lock on the RLS policy's lookup table. --- ============================================================ - -CREATE TABLE copy_rls_lookup (cat INT) DISTRIBUTED BY (cat); -INSERT INTO copy_rls_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_main (a INT, category INT) DISTRIBUTED BY (a); -INSERT INTO copy_rls_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls ON copy_rls_main USING (category IN (SELECT cat FROM copy_rls_lookup)); - --- Create non-superuser to trigger RLS (needs pg_write_server_files to COPY TO file) -CREATE ROLE copy_rls_testuser; -GRANT pg_write_server_files TO copy_rls_testuser; -GRANT ALL ON copy_rls_main TO copy_rls_testuser; -GRANT ALL ON copy_rls_lookup TO copy_rls_testuser; - -SELECT count(*) FROM copy_rls_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Session 1: reorganize the lookup table -1: BEGIN; -1: ALTER TABLE copy_rls_lookup SET WITH (reorganize=true); - --- Session 2: COPY TO as non-superuser (RLS active, internally converted to query-based) -2&: SET ROLE copy_rls_testuser; COPY copy_rls_main TO '/tmp/copy_rls_main.csv'; - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_main%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Reset session 2's role to avoid leaking to subsequent tests -2: RESET ROLE; - --- Verify: should match baseline count (400 rows filtered by RLS) -RESET ROLE; -CREATE TABLE copy_rls_verify (a INT, category INT) DISTRIBUTED BY (a); -COPY copy_rls_verify FROM '/tmp/copy_rls_main.csv'; -SELECT count(*) FROM copy_rls_verify; - --- Cleanup -DROP TABLE copy_rls_verify; -DROP POLICY p_rls ON copy_rls_main; -DROP TABLE copy_rls_main; -DROP TABLE copy_rls_lookup; -DROP ROLE copy_rls_testuser; - --- ============================================================ --- Test 2.5: CTAS + concurrent reorganize --- Fixed as a side effect: CTAS goes through pg_analyze_and_rewrite() + --- AcquireRewriteLocks(), so the snapshot refresh in BeginCopy() also fixes it. --- ============================================================ - -CREATE TABLE ctas_reorg_src (a INT, b INT) DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_src; - --- Session 1: reorganize -1: BEGIN; -1: ALTER TABLE ctas_reorg_src SET WITH (reorganize=true); - --- Session 2: CTAS should block (lock acquired in executor or analyze phase) -2&: CREATE TABLE ctas_reorg_dst AS SELECT * FROM ctas_reorg_src DISTRIBUTED BY (a); - --- Confirm Session 2 is blocked -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_dst%' AND wait_event_type = 'Lock'; - --- Session 1: Commit -1: COMMIT; - --- Session 2: Complete -2<: - --- Verify row count after CTAS completes -SELECT count(*) FROM ctas_reorg_dst; - --- Cleanup -DROP TABLE ctas_reorg_dst; -DROP TABLE ctas_reorg_src; - --- NOTE: Test 2.6 (change distribution key + query-based COPY TO) removed because --- ALTER TABLE SET DISTRIBUTED BY + concurrent query-based COPY TO causes a server --- crash (pre-existing Cloudberry bug, not related to this fix). - --- ============================================================ --- Test 2.1a: AO row table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized row-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_row_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_ao_row_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_row_test TO '/tmp/copy_reorg_ao_row_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_reorg_ao_row_verify FROM '/tmp/copy_reorg_ao_row_test.csv'; -SELECT count(*) FROM copy_reorg_ao_row_verify; - --- Cleanup -DROP TABLE copy_reorg_ao_row_verify; -DROP TABLE copy_reorg_ao_row_test; - --- ============================================================ --- Test 2.1b: AO column table — relation-based COPY TO + concurrent reorganize --- Same as 2.1 but using append-optimized column-oriented table. --- ============================================================ - -CREATE TABLE copy_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; - --- Record original row count -SELECT count(*) FROM copy_reorg_ao_col_test; - --- Session 1: Begin reorganize (holds AccessExclusiveLock) -1: BEGIN; -1: ALTER TABLE copy_reorg_ao_col_test SET WITH (reorganize=true); - --- Session 2: relation-based COPY TO should block on AccessShareLock -2&: COPY copy_reorg_ao_col_test TO '/tmp/copy_reorg_ao_col_test.csv'; - --- Confirm Session 2 is waiting for the lock -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - --- Session 1: Commit reorganize, releasing AccessExclusiveLock -1: COMMIT; - --- Session 2: Should return 1000 rows (fixed), not 0 rows (broken) -2<: - --- Verify the output file contains all rows -CREATE TABLE copy_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_reorg_ao_col_verify FROM '/tmp/copy_reorg_ao_col_test.csv'; -SELECT count(*) FROM copy_reorg_ao_col_verify; - --- Cleanup -DROP TABLE copy_reorg_ao_col_verify; -DROP TABLE copy_reorg_ao_col_test; - --- ============================================================ --- Test 2.2a: AO row — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_row_test (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_ao_row_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_ao_row_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_ao_row_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_ao_row_test) TO '/tmp/copy_query_reorg_ao_row_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_row_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_query_reorg_ao_row_verify FROM '/tmp/copy_query_reorg_ao_row_test.csv'; -SELECT count(*) FROM copy_query_reorg_ao_row_verify; - -DROP TABLE copy_query_reorg_ao_row_verify; -DROP TABLE copy_query_reorg_ao_row_test; - --- ============================================================ --- Test 2.2b: AO column — query-based COPY TO + concurrent reorganize --- Fixed: BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_query_reorg_ao_col_test (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_query_reorg_ao_col_test SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_query_reorg_ao_col_test; - -1: BEGIN; -1: ALTER TABLE copy_query_reorg_ao_col_test SET WITH (reorganize=true); - -2&: COPY (SELECT * FROM copy_query_reorg_ao_col_test) TO '/tmp/copy_query_reorg_ao_col_test.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY (SELECT%copy_query_reorg_ao_col_test%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_query_reorg_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_query_reorg_ao_col_verify FROM '/tmp/copy_query_reorg_ao_col_test.csv'; -SELECT count(*) FROM copy_query_reorg_ao_col_verify; - -DROP TABLE copy_query_reorg_ao_col_verify; -DROP TABLE copy_query_reorg_ao_col_test; - --- ============================================================ --- Test 2.3a: AO row — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_row (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (1) TO (501) USING ao_row; -CREATE TABLE copy_part_child2_ao_row PARTITION OF copy_part_parent_ao_row FOR VALUES FROM (501) TO (1001) USING ao_row; -INSERT INTO copy_part_parent_ao_row SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_ao_row; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_ao_row SET WITH (reorganize=true); - -2&: COPY copy_part_parent_ao_row TO '/tmp/copy_part_parent_ao_row.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_ao_row%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_ao_row_verify (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_part_ao_row_verify FROM '/tmp/copy_part_parent_ao_row.csv'; -SELECT count(*) FROM copy_part_ao_row_verify; - -DROP TABLE copy_part_ao_row_verify; -DROP TABLE copy_part_parent_ao_row; - --- ============================================================ --- Test 2.3b: AO column — partitioned table COPY TO + child partition concurrent reorganize --- Fixed: DoCopy() calls find_all_inheritors() to lock all child partitions first. --- ============================================================ - -CREATE TABLE copy_part_parent_ao_col (a INT, b INT) PARTITION BY RANGE (a) DISTRIBUTED BY (a); -CREATE TABLE copy_part_child1_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (1) TO (501) USING ao_column; -CREATE TABLE copy_part_child2_ao_col PARTITION OF copy_part_parent_ao_col FOR VALUES FROM (501) TO (1001) USING ao_column; -INSERT INTO copy_part_parent_ao_col SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM copy_part_parent_ao_col; - -1: BEGIN; -1: ALTER TABLE copy_part_child1_ao_col SET WITH (reorganize=true); - -2&: COPY copy_part_parent_ao_col TO '/tmp/copy_part_parent_ao_col.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'COPY copy_part_parent_ao_col%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -CREATE TABLE copy_part_ao_col_verify (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_part_ao_col_verify FROM '/tmp/copy_part_parent_ao_col.csv'; -SELECT count(*) FROM copy_part_ao_col_verify; - -DROP TABLE copy_part_ao_col_verify; -DROP TABLE copy_part_parent_ao_col; - --- ============================================================ --- Test 2.4a: AO row — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_row_lookup (cat INT) USING ao_row DISTRIBUTED BY (cat); -INSERT INTO copy_rls_ao_row_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_ao_row_main (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO copy_rls_ao_row_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_ao_row_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_ao_row ON copy_rls_ao_row_main USING (category IN (SELECT cat FROM copy_rls_ao_row_lookup)); - -CREATE ROLE copy_rls_ao_row_testuser; -GRANT pg_write_server_files TO copy_rls_ao_row_testuser; -GRANT ALL ON copy_rls_ao_row_main TO copy_rls_ao_row_testuser; -GRANT ALL ON copy_rls_ao_row_lookup TO copy_rls_ao_row_testuser; - -SELECT count(*) FROM copy_rls_ao_row_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_ao_row_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_ao_row_testuser; COPY copy_rls_ao_row_main TO '/tmp/copy_rls_ao_row_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_ao_row_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_ao_row_verify (a INT, category INT) USING ao_row DISTRIBUTED BY (a); -COPY copy_rls_ao_row_verify FROM '/tmp/copy_rls_ao_row_main.csv'; -SELECT count(*) FROM copy_rls_ao_row_verify; - -DROP TABLE copy_rls_ao_row_verify; -DROP POLICY p_rls_ao_row ON copy_rls_ao_row_main; -DROP TABLE copy_rls_ao_row_main; -DROP TABLE copy_rls_ao_row_lookup; -DROP ROLE copy_rls_ao_row_testuser; - --- ============================================================ --- Test 2.4b: AO column — RLS table COPY TO + policy-referenced table concurrent reorganize --- Fixed: same as 2.4 — BeginCopy() refreshes snapshot after AcquireRewriteLocks(). --- ============================================================ - -CREATE TABLE copy_rls_ao_col_lookup (cat INT) USING ao_column DISTRIBUTED BY (cat); -INSERT INTO copy_rls_ao_col_lookup SELECT i FROM generate_series(1, 2) i; - -CREATE TABLE copy_rls_ao_col_main (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO copy_rls_ao_col_main SELECT i, (i % 5) + 1 FROM generate_series(1, 1000) i; - -ALTER TABLE copy_rls_ao_col_main ENABLE ROW LEVEL SECURITY; -CREATE POLICY p_rls_ao_col ON copy_rls_ao_col_main USING (category IN (SELECT cat FROM copy_rls_ao_col_lookup)); - -CREATE ROLE copy_rls_ao_col_testuser; -GRANT pg_write_server_files TO copy_rls_ao_col_testuser; -GRANT ALL ON copy_rls_ao_col_main TO copy_rls_ao_col_testuser; -GRANT ALL ON copy_rls_ao_col_lookup TO copy_rls_ao_col_testuser; - -SELECT count(*) FROM copy_rls_ao_col_main; - --- Baseline: verify RLS filters correctly (should return 400 rows: categories 1 and 2 only) -2: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: BEGIN; -1: ALTER TABLE copy_rls_ao_col_lookup SET WITH (reorganize=true); - -2&: SET ROLE copy_rls_ao_col_testuser; COPY copy_rls_ao_col_main TO '/tmp/copy_rls_ao_col_main.csv'; - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE '%COPY copy_rls_ao_col_main%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -2: RESET ROLE; - -RESET ROLE; -CREATE TABLE copy_rls_ao_col_verify (a INT, category INT) USING ao_column DISTRIBUTED BY (a); -COPY copy_rls_ao_col_verify FROM '/tmp/copy_rls_ao_col_main.csv'; -SELECT count(*) FROM copy_rls_ao_col_verify; - -DROP TABLE copy_rls_ao_col_verify; -DROP POLICY p_rls_ao_col ON copy_rls_ao_col_main; -DROP TABLE copy_rls_ao_col_main; -DROP TABLE copy_rls_ao_col_lookup; -DROP ROLE copy_rls_ao_col_testuser; - --- ============================================================ --- Test 2.5a: AO row — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_row_src (a INT, b INT) USING ao_row DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_ao_row_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_ao_row_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_ao_row_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_ao_row_dst AS SELECT * FROM ctas_reorg_ao_row_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_row_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_ao_row_dst; - -DROP TABLE ctas_reorg_ao_row_dst; -DROP TABLE ctas_reorg_ao_row_src; - --- ============================================================ --- Test 2.5b: AO column — CTAS + concurrent reorganize --- Fixed as a side effect via BeginCopy() snapshot refresh. --- ============================================================ - -CREATE TABLE ctas_reorg_ao_col_src (a INT, b INT) USING ao_column DISTRIBUTED BY (a); -INSERT INTO ctas_reorg_ao_col_src SELECT i, i FROM generate_series(1, 1000) i; - -SELECT count(*) FROM ctas_reorg_ao_col_src; - -1: BEGIN; -1: ALTER TABLE ctas_reorg_ao_col_src SET WITH (reorganize=true); - -2&: CREATE TABLE ctas_reorg_ao_col_dst AS SELECT * FROM ctas_reorg_ao_col_src DISTRIBUTED BY (a); - -1: SELECT count(*) > 0 FROM pg_stat_activity - WHERE query LIKE 'CREATE TABLE ctas_reorg_ao_col_dst%' AND wait_event_type = 'Lock'; - -1: COMMIT; -2<: - -SELECT count(*) FROM ctas_reorg_ao_col_dst; - -DROP TABLE ctas_reorg_ao_col_dst; -DROP TABLE ctas_reorg_ao_col_src; - --- NOTE: Tests 2.6a/2.6b (AO variants of change distribution key + query-based COPY TO) --- removed for the same reason as test 2.6 (server crash, pre-existing bug). From 30999f178e32863a7ce1196d0a090bc6ea2af232 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 23 Mar 2023 15:15:26 +0300 Subject: [PATCH 058/167] [yagp_hooks_collector] Add extension skeleton with GRPC transport Add yagp_hooks_collector, a shared-preload module that hooks into ExecutorStart and ExecutorFinish to capture query lifecycle events. Includes Makefile with protobuf code generation, GRPC-based delivery, QueryInfo generation (query text, plan text, query_id, plan_id, session metadata), and basic protobuf message filling. --- .gitignore | 7 +- Makefile | 2 - protos/yagpcc_metrics.proto | 130 +++ protos/yagpcc_plan.proto | 570 +++++++++++++ protos/yagpcc_set_service.proto | 45 + sql/yagp-hooks-collector--1.0.sql | 2 + sql/yagp-hooks-collector--unpackaged--1.0.sql | 2 + src/EventSender.cpp | 189 +++++ src/EventSender.h | 19 + src/GrpcConnector.cpp | 55 ++ src/GrpcConnector.h | 15 + src/hook_wrappers.cpp | 67 ++ src/hook_wrappers.h | 12 + src/stat_statements_parser/README.MD | 1 + .../pg_stat_statements_ya_parser.c | 771 ++++++++++++++++++ .../pg_stat_statements_ya_parser.h | 15 + src/yagp_hooks_collector.c | 22 + yagp-hooks-collector.control | 5 + 18 files changed, 1926 insertions(+), 3 deletions(-) create mode 100644 protos/yagpcc_metrics.proto create mode 100644 protos/yagpcc_plan.proto create mode 100644 protos/yagpcc_set_service.proto create mode 100644 sql/yagp-hooks-collector--1.0.sql create mode 100644 sql/yagp-hooks-collector--unpackaged--1.0.sql create mode 100644 src/EventSender.cpp create mode 100644 src/EventSender.h create mode 100644 src/GrpcConnector.cpp create mode 100644 src/GrpcConnector.h create mode 100644 src/hook_wrappers.cpp create mode 100644 src/hook_wrappers.h create mode 100644 src/stat_statements_parser/README.MD create mode 100644 src/stat_statements_parser/pg_stat_statements_ya_parser.c create mode 100644 src/stat_statements_parser/pg_stat_statements_ya_parser.h create mode 100644 src/yagp_hooks_collector.c create mode 100644 yagp-hooks-collector.control diff --git a/.gitignore b/.gitignore index 5c21989c4ab..29b40ee096c 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,9 @@ lib*.pc /compile_commands.json /tmp_install/ /.cache/ -/install/ \ No newline at end of file +/install/ +*.o +*.so +src/protos/ +.vscode +compile_commands.json diff --git a/Makefile b/Makefile index e9ab3fbf2d4..15c5dabb70e 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,12 @@ # to build Postgres with a different make, we have this make file # that, as a service, will look for a GNU make and invoke it, or show # an error message if none could be found. - # If the user were using GNU make now, this file would not get used # because GNU make uses a make file named "GNUmakefile" in preference # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. - # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. all: diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto new file mode 100644 index 00000000000..b7e255484c7 --- /dev/null +++ b/protos/yagpcc_metrics.proto @@ -0,0 +1,130 @@ +syntax = "proto3"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCM"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; + +enum QueryStatus { + QUERY_STATUS_UNSPECIFIED = 0; + QUERY_STATUS_SUBMIT = 1; + QUERY_STATUS_START = 2; + QUERY_STATUS_DONE = 3; + QUERY_STATUS_QUERY_DONE = 4; + QUERY_STATUS_ERROR = 5; + QUERY_STATUS_CANCELLING = 6; + QUERY_STATUS_CANCELED = 7; + QUERY_STATUS_END = 8; +} + +enum PlanNodeStatus { + PLAN_NODE_STATUS_UNSPECIFIED = 0; + PLAN_NODE_STATUS_INITIALIZED = 1; + PLAN_NODE_STATUS_EXECUTING = 2; + PLAN_NODE_STATUS_FINISHED = 3; +} + +message QueryInfo { + PlanGenerator generator = 1; + uint64 query_id = 2; + uint64 plan_id = 3; + string queryText = 4; + string planText = 5; + SessionInfo sessionInfo = 6; +} + +enum PlanGenerator +{ + PLAN_GENERATOR_UNSPECIFIED = 0; + PLAN_GENERATOR_PLANNER = 1; /* plan produced by the planner*/ + PLAN_GENERATOR_OPTIMIZER = 2; /* plan produced by the optimizer*/ +} + +message GPMetrics { + SystemStat systemStat = 1; + MetricInstrumentation instrumentation = 2; + SpillInfo spill = 3; +} + +message QueryInfoHeader { + int32 pid = 1; + GpId gpIdentity = 2; + + int32 tmid = 3; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ + int32 ssid = 4; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ + int32 ccnt = 5; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ + int32 sliceid = 6; /* slice identificator, 0 means general info for the whole query */ +} + +message GpId { + int32 dbid = 1; /* the dbid of this database */ + int32 segindex = 2; /* content indicator: -1 for entry database, + * 0, ..., n-1 for segment database * + * a primary and its mirror have the same segIndex */ + GpRole gp_role = 3; + GpRole gp_session_role = 4; +} + +enum GpRole +{ + GP_ROLE_UNSPECIFIED = 0; + GP_ROLE_UTILITY = 1; /* Operating as a simple database engine */ + GP_ROLE_DISPATCH = 2; /* Operating as the parallel query dispatcher */ + GP_ROLE_EXECUTE = 3; /* Operating as a parallel query executor */ + GP_ROLE_UNDEFINED = 4; /* Should never see this role in use */ +} + +message SessionInfo { + string sql = 1; + string userName = 2; + string databaseName = 3; + string resourceGroup = 4; + string applicationName = 5; +} + +message SystemStat { + /* CPU stat*/ + double runningTimeSeconds = 1; + double userTimeSeconds = 2; + double kernelTimeSeconds = 3; + + /* Memory stat */ + uint64 vsize = 4; + uint64 rss = 5; + uint64 VmSizeKb = 6; + uint64 VmPeakKb = 7; + + /* Storage stat */ + uint64 rchar = 8; + uint64 wchar = 9; + uint64 syscr = 10; + uint64 syscw = 11; + uint64 read_bytes = 12; + uint64 write_bytes = 13; + uint64 cancelled_write_bytes = 14; +} + +message MetricInstrumentation { + uint64 ntuples = 1; /* Total tuples produced */ + uint64 nloops = 2; /* # of run cycles for this node */ + uint64 tuplecount = 3; /* Tuples emitted so far this cycle */ + double firsttuple = 4; /* Time for first tuple of this cycle */ + double startup = 5; /* Total startup time (in seconds) */ + double total = 6; /* Total total time (in seconds) */ + uint64 shared_blks_hit = 7; /* shared blocks stats*/ + uint64 shared_blks_read = 8; + uint64 shared_blks_dirtied = 9; + uint64 shared_blks_written = 10; + uint64 local_blks_hit = 11; /* data read from disks */ + uint64 local_blks_read = 12; + uint64 local_blks_dirtied = 13; + uint64 local_blks_written = 14; + uint64 temp_blks_read = 15; /* temporary tables read stat */ + uint64 temp_blks_written = 16; + double blk_read_time = 17; /* measured read/write time */ + double blk_write_time = 18; +} + +message SpillInfo { + int32 fileCount = 1; + int64 totalBytes = 2; +} diff --git a/protos/yagpcc_plan.proto b/protos/yagpcc_plan.proto new file mode 100644 index 00000000000..962fab4bbdd --- /dev/null +++ b/protos/yagpcc_plan.proto @@ -0,0 +1,570 @@ +syntax = "proto3"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCP"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; + +message MetricPlan { + GpdbNodeType type = 1; + + int32 plan_node_id = 2; + int32 parent_plan_node_id = 3; // Valid only for QueryInfoMetricQuerySubmit + + double startup_cost = 4; /* cost expended before fetching any tuples */ + double total_cost = 5; /* total cost (assuming all tuples fetched) */ + double plan_rows = 6; /* number of rows plan is expected to emit */ + int32 plan_width = 7; /* average row width in bytes */ + + int32 arg1 = 8; // for some nodes it's additional opperand type + int32 arg2 = 9; // for some nodes it's additional opperand type + + MetricMotionInfo motion_info = 10; + MetricRelationInfo relation_info = 11; + + string scan_index_name = 12; + ScanDirection scan_direction = 13; + MetricSliceInfo slice_info = 14; + string statement = 15; +} + +message MetricMotionInfo { + MotionType type = 1; + bool isBroadcast = 2; + CdbLocusType locusType = 3; + + int32 sliceId = 4; + int32 parentSliceId = 5; +} + +message MetricRelationInfo { + int32 oid = 1; + string name = 2; + string schema = 3; + string alias = 4; + int32 dynamicScanId = 5; +} + +message MetricSliceInfo { + int32 slice = 1; + int32 segments = 2; + GangType gangType = 3; + int32 gang = 4; +} + +enum ScanDirection +{ + SCAN_DIRECTION_UNSPECIFIED = 0; + SCAN_DIRECTION_BACKWARD = 1; + SCAN_DIRECTION_FORWARD = 2; +} + +/* GangType enumeration is used in several structures related to CDB + * slice plan support. + */ +enum GangType +{ + GANG_TYPE_UNSPECIFIED = 0; + GANG_TYPE_UNALLOCATED = 1; /* a root slice executed by the qDisp */ + GANG_TYPE_ENTRYDB_READER = 2; /* a 1-gang with read access to the entry db */ + GANG_TYPE_SINGLETON_READER = 3; /* a 1-gang to read the segment dbs */ + GANG_TYPE_PRIMARY_READER = 4; /* a 1-gang or N-gang to read the segment dbs */ + GANG_TYPE_PRIMARY_WRITER = 5; /* the N-gang that can update the segment dbs */ +} + + +enum CdbLocusType +{ + CDB_LOCUS_TYPE_UNSPECIFIED = 0; + CDB_LOCUS_TYPE_ENTRY = 1; /* a single backend process on the entry db: + * usually the qDisp itself, but could be a + * qExec started by the entry postmaster. + */ + + CDB_LOCUS_TYPE_SINGLE_QE = 2; /* a single backend process on any db: the + * qDisp itself, or a qExec started by a + * segment postmaster or the entry postmaster. + */ + + CDB_LOCUS_TYPE_GENERAL = 3; /* compatible with any locus (data is + * self-contained in the query plan or + * generally available in any qExec or qDisp) */ + + CDB_LOCUS_TYPE_SEGMENT_GENERAL = 4; /* generally available in any qExec, but not + * available in qDisp */ + + CDB_LOCUS_TYPE_REPLICATED = 5; /* replicated over all qExecs of an N-gang */ + CDB_LOCUS_TYPE_HASHED = 6; /* hash partitioned over all qExecs of N-gang */ + CDB_LOCUS_TYPE_HASHED_OJ = 7; /* result of hash partitioned outer join, NULLs can be anywhere */ + CDB_LOCUS_TYPE_STREWN = 8; /* partitioned on no known function */ + CDB_LOCUS_TYPE_END = 9; /* = last valid CdbLocusType + 1 */ +} + +enum MotionType +{ + MOTION_TYPE_UNSPECIFIED = 0; + MOTION_TYPE_HASH = 1; // Use hashing to select a segindex destination + MOTION_TYPE_FIXED = 2; // Send tuples to a fixed set of segindexes + MOTION_TYPE_EXPLICIT = 3; // Send tuples to the segment explicitly specified in their segid column +} + +enum GpdbNodeType { + GPDB_NODE_TYPE_UNSPECIFIED = 0; + INDEX_INFO = 1; + EXPR_CONTEXT = 2; + PROJECTION_INFO = 3; + JUNK_FILTER = 4; + RESULT_REL_INFO = 5; + E_STATE = 6; + TUPLE_TABLE_SLOT = 7; + CDB_PROCESS = 8; + SLICE = 9; + SLICE_TABLE = 10; + CURSOR_POS_INFO = 11; + SHARE_NODE_ENTRY = 12; + PARTITION_STATE = 13; + QUERY_DISPATCH_DESC = 14; + OID_ASSIGNMENT = 15; + PLAN = 16; + SCAN = 17; + JOIN = 18; + RESULT = 19; + MODIFY_TABLE = 20; + APPEND = 21; + MERGE_APPEND = 22; + RECURSIVE_UNION = 23; + SEQUENCE = 24; + BITMAP_AND = 25; + BITMAP_OR = 26; + SEQ_SCAN = 27; + DYNAMIC_SEQ_SCAN = 28; + EXTERNAL_SCAN = 29; + INDEX_SCAN = 30; + DYNAMIC_INDEX_SCAN = 31; + INDEX_ONLY_SCAN = 32; + BITMAP_INDEX_SCAN = 33; + DYNAMIC_BITMAP_INDEX_SCAN = 34; + BITMAP_HEAP_SCAN = 35; + DYNAMIC_BITMAP_HEAP_SCAN = 36; + TID_SCAN = 37; + SUBQUERY_SCAN = 38; + FUNCTION_SCAN = 39; + TABLE_FUNCTION_SCAN = 40; + VALUES_SCAN = 41; + CTE_SCAN = 42; + WORK_TABLE_SCAN = 43; + FOREIGN_SCAN = 44; + NEST_LOOP = 45; + MERGE_JOIN = 46; + HASH_JOIN = 47; + MATERIAL = 48; + SORT = 49; + AGG = 50; + WINDOW_AGG = 51; + UNIQUE = 52; + HASH = 53; + SET_OP = 54; + LOCK_ROWS = 55; + LIMIT = 56; + MOTION = 57; + SHARE_INPUT_SCAN = 58; + REPEAT = 59; + DML = 60; + SPLIT_UPDATE = 61; + ROW_TRIGGER = 62; + ASSERT_OP = 63; + PARTITION_SELECTOR = 64; + PLAN_END = 65; + NEST_LOOP_PARAM = 66; + PLAN_ROW_MARK = 67; + PLAN_INVAL_ITEM = 68; + PLAN_STATE = 69; + SCAN_STATE = 70; + JOIN_STATE = 71; + RESULT_STATE = 72; + MODIFY_TABLE_STATE = 73; + APPEND_STATE = 74; + MERGE_APPEND_STATE = 75; + RECURSIVE_UNION_STATE = 76; + SEQUENCE_STATE = 77; + BITMAP_AND_STATE = 78; + BITMAP_OR_STATE = 79; + SEQ_SCAN_STATE = 80; + DYNAMIC_SEQ_SCAN_STATE = 81; + EXTERNAL_SCAN_STATE = 82; + INDEX_SCAN_STATE = 83; + DYNAMIC_INDEX_SCAN_STATE = 84; + INDEX_ONLY_SCAN_STATE = 85; + BITMAP_INDEX_SCAN_STATE = 86; + DYNAMIC_BITMAP_INDEX_SCAN_STATE = 87; + BITMAP_HEAP_SCAN_STATE = 88; + DYNAMIC_BITMAP_HEAP_SCAN_STATE = 89; + TID_SCAN_STATE = 90; + SUBQUERY_SCAN_STATE = 91; + FUNCTION_SCAN_STATE = 92; + TABLE_FUNCTION_STATE = 93; + VALUES_SCAN_STATE = 94; + CTE_SCAN_STATE = 95; + WORK_TABLE_SCAN_STATE = 96; + FOREIGN_SCAN_STATE = 97; + NEST_LOOP_STATE = 98; + MERGE_JOIN_STATE = 99; + HASH_JOIN_STATE = 100; + MATERIAL_STATE = 101; + SORT_STATE = 102; + AGG_STATE = 103; + WINDOW_AGG_STATE = 104; + UNIQUE_STATE = 105; + HASH_STATE = 106; + SET_OP_STATE = 107; + LOCK_ROWS_STATE = 108; + LIMIT_STATE = 109; + MOTION_STATE = 110; + SHARE_INPUT_SCAN_STATE = 111; + REPEAT_STATE = 112; + DML_STATE = 113; + SPLIT_UPDATE_STATE = 114; + ROW_TRIGGER_STATE = 115; + ASSERT_OP_STATE = 116; + PARTITION_SELECTOR_STATE = 117; + TUPLE_DESC_NODE = 118; + SERIALIZED_PARAM_EXTERN_DATA = 119; + ALIAS = 120; + RANGE_VAR = 121; + EXPR = 122; + VAR = 123; + CONST = 124; + PARAM = 125; + AGGREF = 126; + WINDOW_FUNC = 127; + ARRAY_REF = 128; + FUNC_EXPR = 129; + NAMED_ARG_EXPR = 130; + OP_EXPR = 131; + DISTINCT_EXPR = 132; + NULL_IF_EXPR = 133; + SCALAR_ARRAY_OP_EXPR = 134; + BOOL_EXPR = 135; + SUB_LINK = 136; + SUB_PLAN = 137; + ALTERNATIVE_SUB_PLAN = 138; + FIELD_SELECT = 139; + FIELD_STORE = 140; + RELABEL_TYPE = 141; + COERCE_VIA_IO = 142; + ARRAY_COERCE_EXPR = 143; + CONVERT_ROWTYPE_EXPR = 144; + COLLATE_EXPR = 145; + CASE_EXPR = 146; + CASE_WHEN = 147; + CASE_TEST_EXPR = 148; + ARRAY_EXPR = 149; + ROW_EXPR = 150; + ROW_COMPARE_EXPR = 151; + COALESCE_EXPR = 152; + MIN_MAX_EXPR = 153; + XML_EXPR = 154; + NULL_TEST = 155; + BOOLEAN_TEST = 156; + COERCE_TO_DOMAIN = 157; + COERCE_TO_DOMAIN_VALUES = 158; + SET_TO_DEFAULT = 159; + CURRENT_OF_EXPR = 160; + TARGET_ENTRY = 161; + RANGE_TBL_REF = 162; + JOIN_EXPR = 163; + FROM_EXPR = 164; + INTO_CLAUSE = 165; + COPY_INTO_CLAUSE = 166; + REFRESH_CLAUSE = 167; + FLOW = 168; + GROUPING = 169; + GROUP_ID = 170; + DISTRIBUTED_BY = 171; + DML_ACTION_EXPR = 172; + PART_SELECTED_EXPR = 173; + PART_DEFAULT_EXPR = 174; + PART_BOUND_EXPR = 175; + PART_BOUND_INCLUSION_EXPR = 176; + PART_BOUND_OPEN_EXPR = 177; + PART_LIST_RULE_EXPR = 178; + PART_LIST_NULL_TEST_EXPR = 179; + TABLE_OID_INFO = 180; + EXPR_STATE = 181; + GENERIC_EXPR_STATE = 182; + WHOLE_ROW_VAR_EXPR_STATE = 183; + AGGREF_EXPR_STATE = 184; + WINDOW_FUNC_EXPR_STATE = 185; + ARRAY_REF_EXPR_STATE = 186; + FUNC_EXPR_STATE = 187; + SCALAR_ARRAY_OP_EXPR_STATE = 188; + BOOL_EXPR_STATE = 189; + SUB_PLAN_STATE = 190; + ALTERNATIVE_SUB_PLAN_STATE = 191; + FIELD_SELECT_STATE = 192; + FIELD_STORE_STATE = 193; + COERCE_VIA_IO_STATE = 194; + ARRAY_COERCE_EXPR_STATE = 195; + CONVERT_ROWTYPE_EXPR_STATE = 196; + CASE_EXPR_STATE = 197; + CASE_WHEN_STATE = 198; + ARRAY_EXPR_STATE = 199; + ROW_EXPR_STATE = 200; + ROW_COMPARE_EXPR_STATE = 201; + COALESCE_EXPR_STATE = 202; + MIN_MAX_EXPR_STATE = 203; + XML_EXPR_STATE = 204; + NULL_TEST_STATE = 205; + COERCE_TO_DOMAIN_STATE = 206; + DOMAIN_CONSTRAINT_STATE = 207; + GROUPING_FUNC_EXPR_STATE = 208; + PART_SELECTED_EXPR_STATE = 209; + PART_DEFAULT_EXPR_STATE = 210; + PART_BOUND_EXPR_STATE = 211; + PART_BOUND_INCLUSION_EXPR_STATE = 212; + PART_BOUND_OPEN_EXPR_STATE = 213; + PART_LIST_RULE_EXPR_STATE = 214; + PART_LIST_NULL_TEST_EXPR_STATE = 215; + PLANNER_INFO = 216; + PLANNER_GLOBAL = 217; + REL_OPT_INFO = 218; + INDEX_OPT_INFO = 219; + PARAM_PATH_INFO = 220; + PATH = 221; + APPEND_ONLY_PATH = 222; + AOCS_PATH = 223; + EXTERNAL_PATH = 224; + INDEX_PATH = 225; + BITMAP_HEAP_PATH = 226; + BITMAP_AND_PATH = 227; + BITMAP_OR_PATH = 228; + NEST_PATH = 229; + MERGE_PATH = 230; + HASH_PATH = 231; + TID_PATH = 232; + FOREIGN_PATH = 233; + APPEND_PATH = 234; + MERGE_APPEND_PATH = 235; + RESULT_PATH = 236; + MATERIAL_PATH = 237; + UNIQUE_PATH = 238; + PROJECTION_PATH = 239; + EQUIVALENCE_CLASS = 240; + EQUIVALENCE_MEMBER = 241; + PATH_KEY = 242; + RESTRICT_INFO = 243; + PLACE_HOLDER_VAR = 244; + SPECIAL_JOIN_INFO = 245; + LATERAL_JOIN_INFO = 246; + APPEND_REL_INFO = 247; + PLACE_HOLDER_INFO = 248; + MIN_MAX_AGG_INFO = 249; + PARTITION = 250; + PARTITION_RULE = 251; + PARTITION_NODE = 252; + PG_PART_RULE = 253; + SEGFILE_MAP_NODE = 254; + PLANNER_PARAM_ITEM = 255; + CDB_MOTION_PATH = 256; + PARTITION_SELECTOR_PATH = 257; + CDB_REL_COLUMN_INFO = 258; + DISTRIBUTION_KEY = 259; + MEMORY_CONTEXT = 260; + ALLOC_SET_CONTEXT = 261; + MEMORY_ACCOUNT = 262; + VALUE = 263; + INTEGER = 264; + FLOAT = 265; + STRING = 266; + BIT_STRING = 267; + NULL_VALUE = 268; + LIST = 269; + INT_LIST = 270; + OID_LIST = 271; + QUERY = 272; + PLANNED_STMT = 273; + INSERT_STMT = 274; + DELETE_STMT = 275; + UPDATE_STMT = 276; + SELECT_STMT = 277; + ALTER_TABLE_STMT = 278; + ALTER_TABLE_CMD = 279; + ALTER_DOMAIN_STMT = 280; + SET_OPERATION_STMT = 281; + GRANT_STMT = 282; + GRANT_ROLE_STMT = 283; + ALTER_DEFAULT_PRIVILEGES_STMT = 284; + CLOSE_PORTAL_STMT = 285; + CLUSTER_STMT = 286; + COPY_STMT = 287; + CREATE_STMT = 288; + SINGLE_ROW_ERROR_DESC = 289; + EXT_TABLE_TYPE_DESC = 290; + CREATE_EXTERNAL_STMT = 291; + DEFINE_STMT = 292; + DROP_STMT = 293; + TRUNCATE_STMT = 294; + COMMENT_STMT = 295; + FETCH_STMT = 296; + INDEX_STMT = 297; + CREATE_FUNCTION_STMT = 298; + ALTER_FUNCTION_STMT = 299; + DO_STMT = 300; + RENAME_STMT = 301; + RULE_STMT = 302; + NOTIFY_STMT = 303; + LISTEN_STMT = 304; + UNLISTEN_STMT = 305; + TRANSACTION_STMT = 306; + VIEW_STMT = 307; + LOAD_STMT = 308; + CREATE_DOMAIN_STMT = 309; + CREATEDB_STMT = 310; + DROPDB_STMT = 311; + VACUUM_STMT = 312; + EXPLAIN_STMT = 313; + CREATE_TABLE_AS_STMT = 314; + CREATE_SEQ_STMT = 315; + ALTER_SEQ_STMT = 316; + VARIABLE_SET_STMT = 317; + VARIABLE_SHOW_STMT = 318; + DISCARD_STMT = 319; + CREATE_TRIG_STMT = 320; + CREATE_P_LANG_STMT = 321; + CREATE_ROLE_STMT = 322; + ALTER_ROLE_STMT = 323; + DROP_ROLE_STMT = 324; + CREATE_QUEUE_STMT = 325; + ALTER_QUEUE_STMT = 326; + DROP_QUEUE_STMT = 327; + CREATE_RESOURCE_GROUP_STMT = 328; + DROP_RESOURCE_GROUP_STMT = 329; + ALTER_RESOURCE_GROUP_STMT = 330; + LOCK_STMT = 331; + CONSTRAINTS_SET_STMT = 332; + REINDEX_STMT = 333; + CHECK_POINT_STMT = 334; + CREATE_SCHEMA_STMT = 335; + ALTER_DATABASE_STMT = 336; + ALTER_DATABASE_SET_STMT = 337; + ALTER_ROLE_SET_STMT = 338; + CREATE_CONVERSION_STMT = 339; + CREATE_CAST_STMT = 340; + CREATE_OP_CLASS_STMT = 341; + CREATE_OP_FAMILY_STMT = 342; + ALTER_OP_FAMILY_STMT = 343; + PREPARE_STMT = 344; + EXECUTE_STMT = 345; + DEALLOCATE_STMT = 346; + DECLARE_CURSOR_STMT = 347; + CREATE_TABLE_SPACE_STMT = 348; + DROP_TABLE_SPACE_STMT = 349; + ALTER_OBJECT_SCHEMA_STMT = 350; + ALTER_OWNER_STMT = 351; + DROP_OWNED_STMT = 352; + REASSIGN_OWNED_STMT = 353; + COMPOSITE_TYPE_STMT = 354; + CREATE_ENUM_STMT = 355; + CREATE_RANGE_STMT = 356; + ALTER_ENUM_STMT = 357; + ALTER_TS_DICTIONARY_STMT = 358; + ALTER_TS_CONFIGURATION_STMT = 359; + CREATE_FDW_STMT = 360; + ALTER_FDW_STMT = 361; + CREATE_FOREIGN_SERVER_STMT = 362; + ALTER_FOREIGN_SERVER_STMT = 363; + CREATE_USER_MAPPING_STMT = 364; + ALTER_USER_MAPPING_STMT = 365; + DROP_USER_MAPPING_STMT = 366; + ALTER_TABLE_SPACE_OPTIONS_STMT = 367; + ALTER_TABLE_MOVE_ALL_STMT = 368; + SEC_LABEL_STMT = 369; + CREATE_FOREIGN_TABLE_STMT = 370; + CREATE_EXTENSION_STMT = 371; + ALTER_EXTENSION_STMT = 372; + ALTER_EXTENSION_CONTENTS_STMT = 373; + CREATE_EVENT_TRIG_STMT = 374; + ALTER_EVENT_TRIG_STMT = 375; + REFRESH_MAT_VIEW_STMT = 376; + REPLICA_IDENTITY_STMT = 377; + ALTER_SYSTEM_STMT = 378; + PARTITION_BY = 379; + PARTITION_ELEM = 380; + PARTITION_RANGE_ITEM = 381; + PARTITION_BOUND_SPEC = 382; + PARTITION_SPEC = 383; + PARTITION_VALUES_SPEC = 384; + ALTER_PARTITION_ID = 385; + ALTER_PARTITION_CMD = 386; + INHERIT_PARTITION_CMD = 387; + CREATE_FILE_SPACE_STMT = 388; + FILE_SPACE_ENTRY = 389; + DROP_FILE_SPACE_STMT = 390; + TABLE_VALUE_EXPR = 391; + DENY_LOGIN_INTERVAL = 392; + DENY_LOGIN_POINT = 393; + ALTER_TYPE_STMT = 394; + SET_DISTRIBUTION_CMD = 395; + EXPAND_STMT_SPEC = 396; + A_EXPR = 397; + COLUMN_REF = 398; + PARAM_REF = 399; + A_CONST = 400; + FUNC_CALL = 401; + A_STAR = 402; + A_INDICES = 403; + A_INDIRECTION = 404; + A_ARRAY_EXPR = 405; + RES_TARGET = 406; + TYPE_CAST = 407; + COLLATE_CLAUSE = 408; + SORT_BY = 409; + WINDOW_DEF = 410; + RANGE_SUBSELECT = 411; + RANGE_FUNCTION = 412; + TYPE_NAME = 413; + COLUMN_DEF = 414; + INDEX_ELEM = 415; + CONSTRAINT = 416; + DEF_ELEM = 417; + RANGE_TBL_ENTRY = 418; + RANGE_TBL_FUNCTION = 419; + WITH_CHECK_OPTION = 420; + GROUPING_CLAUSE = 421; + GROUPING_FUNC = 422; + SORT_GROUP_CLAUSE = 423; + WINDOW_CLAUSE = 424; + PRIV_GRANTEE = 425; + FUNC_WITH_ARGS = 426; + ACCESS_PRIV = 427; + CREATE_OP_CLASS_ITEM = 428; + TABLE_LIKE_CLAUSE = 429; + FUNCTION_PARAMETER = 430; + LOCKING_CLAUSE = 431; + ROW_MARK_CLAUSE = 432; + XML_SERIALIZE = 433; + WITH_CLAUSE = 434; + COMMON_TABLE_EXPR = 435; + COLUMN_REFERENCE_STORAGE_DIRECTIVE = 436; + IDENTIFY_SYSTEM_CMD = 437; + BASE_BACKUP_CMD = 438; + CREATE_REPLICATION_SLOT_CMD = 439; + DROP_REPLICATION_SLOT_CMD = 440; + START_REPLICATION_CMD = 441; + TIME_LINE_HISTORY_CMD = 442; + TRIGGER_DATA = 443; + EVENT_TRIGGER_DATA = 444; + RETURN_SET_INFO = 445; + WINDOW_OBJECT_DATA = 446; + TID_BITMAP = 447; + INLINE_CODE_BLOCK = 448; + FDW_ROUTINE = 449; + STREAM_BITMAP = 450; + FORMATTER_DATA = 451; + EXT_PROTOCOL_DATA = 452; + EXT_PROTOCOL_VALIDATOR_DATA = 453; + SELECTED_PARTS = 454; + COOKED_CONSTRAINT = 455; + CDB_EXPLAIN_STAT_HDR = 456; + GP_POLICY = 457; + RETRIEVE_STMT = 458; +} diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto new file mode 100644 index 00000000000..0bef72891ee --- /dev/null +++ b/protos/yagpcc_set_service.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +import "protos/yagpcc_metrics.proto"; +import "protos/yagpcc_plan.proto"; + +package yagpcc; +option java_outer_classname = "SegmentYAGPCCAS"; +option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; + +service SetQueryInfo { + rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} + + rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} +} + +message MetricResponse { + MetricResponseStatusCode error_code = 1; + string error_text = 2; +} + +enum MetricResponseStatusCode { + METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; + METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; + METRIC_RESPONSE_STATUS_CODE_ERROR = 2; +} + +message SetQueryReq { + QueryStatus query_status = 1; + google.protobuf.Timestamp datetime = 2; + + QueryInfoHeader header = 3; + QueryInfo query_info = 4; + GPMetrics query_metrics = 5; + repeated MetricPlan plan_tree = 6; +} + +message SetPlanNodeReq { + PlanNodeStatus node_status = 1; + google.protobuf.Timestamp datetime = 2; + QueryInfoHeader header = 3; + GPMetrics node_metrics = 4; + MetricPlan plan_node = 5; +} diff --git a/sql/yagp-hooks-collector--1.0.sql b/sql/yagp-hooks-collector--1.0.sql new file mode 100644 index 00000000000..f9ab15fb400 --- /dev/null +++ b/sql/yagp-hooks-collector--1.0.sql @@ -0,0 +1,2 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "yagp-hooks-collector"''' to load this file. \quit diff --git a/sql/yagp-hooks-collector--unpackaged--1.0.sql b/sql/yagp-hooks-collector--unpackaged--1.0.sql new file mode 100644 index 00000000000..0441c97bd84 --- /dev/null +++ b/sql/yagp-hooks-collector--unpackaged--1.0.sql @@ -0,0 +1,2 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "uuid-cb" FROM unpackaged''' to load this file. \quit diff --git a/src/EventSender.cpp b/src/EventSender.cpp new file mode 100644 index 00000000000..bb4765adeb1 --- /dev/null +++ b/src/EventSender.cpp @@ -0,0 +1,189 @@ +#include "EventSender.h" +#include "GrpcConnector.h" +#include "protos/yagpcc_set_service.pb.h" +#include + +extern "C" +{ +#include "postgres.h" +#include "utils/metrics_utils.h" +#include "utils/elog.h" +#include "executor/executor.h" +#include "commands/explain.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" + +#include "cdb/cdbvars.h" +#include "cdb/cdbexplain.h" + +#include "tcop/utility.h" +#include "pg_stat_statements_ya_parser.h" +} + +namespace +{ +std::string* get_user_name() +{ + const char *username = GetConfigOption("session_authorization", false, false); + return username ? new std::string(username) : nullptr; +} + +std::string* get_db_name() +{ + char *dbname = get_database_name(MyDatabaseId); + std::string* result = dbname ? new std::string(dbname) : nullptr; + pfree(dbname); + return result; +} + +std::string* get_rg_name() +{ + auto userId = GetUserId(); + if (!OidIsValid(userId)) + return nullptr; + auto groupId = GetResGroupIdForRole(userId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + pfree(rgname); + return new std::string(rgname); +} + +std::string* get_app_name() +{ + return application_name ? new std::string(application_name) : nullptr; +} + +int get_cur_slice_id(QueryDesc *desc) +{ + if (!desc->estate) + { + return 0; + } + return LocallyExecutingSliceIndex(desc->estate); +} + +google::protobuf::Timestamp current_ts() +{ + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_header(yagpcc::QueryInfoHeader *header, QueryDesc *queryDesc) +{ + header->set_pid(MyProcPid); + auto gpId = header->mutable_gpidentity(); + gpId->set_dbid(GpIdentity.dbid); + gpId->set_segindex(GpIdentity.segindex); + gpId->set_gp_role(static_cast(Gp_role)); + gpId->set_gp_session_role(static_cast(Gp_session_role)); + header->set_ssid(gp_session_id); + header->set_ccnt(gp_command_count); + header->set_sliceid(get_cur_slice_id(queryDesc)); + int32 tmid = 0; + gpmon_gettmid(&tmid); + header->set_tmid(tmid); +} + +void set_session_info(yagpcc::SessionInfo *si, QueryDesc *queryDesc) +{ + if (queryDesc->sourceText) + *si->mutable_sql() = std::string(queryDesc->sourceText); + si->set_allocated_applicationname(get_app_name()); + si->set_allocated_databasename(get_db_name()); + si->set_allocated_resourcegroup(get_rg_name()); + si->set_allocated_username(get_user_name()); +} + +ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) +{ + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, queryDesc); + ExplainEndOutput(&es); + return es; +} + +void set_plan_text(std::string *plan_text, QueryDesc *queryDesc) +{ + auto es = get_explain_state(queryDesc, true); + *plan_text = std::string(es.str->data, es.str->len); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) +{ + set_session_info(qi->mutable_sessioninfo(), queryDesc); + if (queryDesc->sourceText) + *qi->mutable_querytext() = queryDesc->sourceText; + if (queryDesc->plannedstmt) + { + qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plantext(), queryDesc); + qi->set_plan_id(get_plan_id(queryDesc)); + qi->set_query_id(queryDesc->plannedstmt->queryId); + } +} +} // namespace + +void EventSender::ExecutorStart(QueryDesc *queryDesc, int /* eflags*/) +{ + elog(DEBUG1, "Query %s start recording", queryDesc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *req.mutable_datetime() = current_ts(); + set_header(req.mutable_header(), queryDesc); + set_query_info(req.mutable_query_info(), queryDesc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) + { + elog(WARNING, "Query %s start reporting failed with an error %s", + queryDesc->sourceText, result.error_text().c_str()); + } + else + { + elog(DEBUG1, "Query %s start successful", queryDesc->sourceText); + } +} + +void EventSender::ExecutorFinish(QueryDesc *queryDesc) +{ + elog(DEBUG1, "Query %s finish recording", queryDesc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + *req.mutable_datetime() = current_ts(); + set_header(req.mutable_header(), queryDesc); + set_query_info(req.mutable_query_info(), queryDesc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) + { + elog(WARNING, "Query %s finish reporting failed with an error %s", + queryDesc->sourceText, result.error_text().c_str()); + } + else + { + elog(DEBUG1, "Query %s finish successful", queryDesc->sourceText); + } +} + +EventSender *EventSender::instance() +{ + static EventSender sender; + return &sender; +} + +EventSender::EventSender() +{ + connector = std::make_unique(); +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h new file mode 100644 index 00000000000..70868f6c757 --- /dev/null +++ b/src/EventSender.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +class GrpcConnector; + +struct QueryDesc; + +class EventSender +{ +public: + void ExecutorStart(QueryDesc *queryDesc, int eflags); + void ExecutorFinish(QueryDesc *queryDesc); + static EventSender *instance(); + +private: + EventSender(); + std::unique_ptr connector; +}; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp new file mode 100644 index 00000000000..7329f392010 --- /dev/null +++ b/src/GrpcConnector.cpp @@ -0,0 +1,55 @@ +#include "GrpcConnector.h" +#include "yagpcc_set_service.grpc.pb.h" + +#include +#include +#include + +class GrpcConnector::Impl +{ +public: + Impl() + { + GOOGLE_PROTOBUF_VERIFY_VERSION; + this->stub = yagpcc::SetQueryInfo::NewStub(grpc::CreateChannel( + SOCKET_FILE, grpc::InsecureChannelCredentials())); + } + + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) + { + yagpcc::MetricResponse response; + grpc::ClientContext context; + auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(50); + context.set_deadline(deadline); + + grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); + + if (!status.ok()) + { + response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + } + + return response; + } + +private: + const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; + const std::string TCP_ADDRESS = "127.0.0.1:1432"; + std::unique_ptr stub; +}; + +GrpcConnector::GrpcConnector() +{ + impl = new Impl(); +} + +GrpcConnector::~GrpcConnector() +{ + delete impl; +} + +yagpcc::MetricResponse GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) +{ + return impl->set_metric_query(req); +} \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h new file mode 100644 index 00000000000..dc0f21706a3 --- /dev/null +++ b/src/GrpcConnector.h @@ -0,0 +1,15 @@ +#pragma once + +#include "yagpcc_set_service.pb.h" + +class GrpcConnector +{ +public: + GrpcConnector(); + ~GrpcConnector(); + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + +private: + class Impl; + Impl *impl; +}; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp new file mode 100644 index 00000000000..9f3200c006f --- /dev/null +++ b/src/hook_wrappers.cpp @@ -0,0 +1,67 @@ +#include "hook_wrappers.h" +#include "EventSender.h" + +extern "C" +{ +#include "postgres.h" +#include "utils/metrics_utils.h" +#include "utils/elog.h" +#include "executor/executor.h" + +#include "cdb/cdbvars.h" +#include "cdb/cdbexplain.h" + +#include "tcop/utility.h" +} + +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" + +static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; +static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; + +static void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags); +static void ya_ExecutorFinish_hook(QueryDesc *queryDesc); + +#define REPLACE_HOOK(hookName) \ + previous_##hookName = hookName; \ + hookName = ya_##hookName; + +void hooks_init() +{ + REPLACE_HOOK(ExecutorStart_hook); + REPLACE_HOOK(ExecutorFinish_hook); + stat_statements_parser_init(); +} + +void hooks_deinit() +{ + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorFinish_hook = ExecutorFinish_hook; + stat_statements_parser_deinit(); +} + +#define CREATE_HOOK_WRAPPER(hookName, ...) \ + PG_TRY(); \ + { \ + EventSender::instance()->hookName(__VA_ARGS__); \ + } \ + PG_CATCH(); \ + { \ + ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ + PG_RE_THROW(); \ + } \ + PG_END_TRY(); \ + if (previous_##hookName##_hook) \ + (*previous_##hookName##_hook)(__VA_ARGS__); \ + else \ + standard_##hookName(__VA_ARGS__); + +void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags) +{ + CREATE_HOOK_WRAPPER(ExecutorStart, queryDesc, eflags); +} + +void ya_ExecutorFinish_hook(QueryDesc *queryDesc) +{ + CREATE_HOOK_WRAPPER(ExecutorFinish, queryDesc); +} \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h new file mode 100644 index 00000000000..815fcb7cd51 --- /dev/null +++ b/src/hook_wrappers.h @@ -0,0 +1,12 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +extern void hooks_init(); +extern void hooks_deinit(); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/src/stat_statements_parser/README.MD b/src/stat_statements_parser/README.MD new file mode 100644 index 00000000000..291e31a3099 --- /dev/null +++ b/src/stat_statements_parser/README.MD @@ -0,0 +1 @@ +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c new file mode 100644 index 00000000000..f14742337bd --- /dev/null +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -0,0 +1,771 @@ +#include "postgres.h" + +#include +#include + +#include "access/hash.h" +#include "executor/instrument.h" +#include "executor/execdesc.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "parser/analyze.h" +#include "parser/parsetree.h" +#include "parser/scanner.h" +#include "parser/gram.h" +#include "pgstat.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/spin.h" +#include "tcop/utility.h" +#include "utils/builtins.h" +#include "utils/memutils.h" + +#include "pg_stat_statements_ya_parser.h" + +static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; + +#define JUMBLE_SIZE 1024 /* query serialization buffer size */ + +/* + * Struct for tracking locations/lengths of constants during normalization + */ +typedef struct pgssLocationLen +{ + int location; /* start offset in query text */ + int length; /* length in bytes, or -1 to ignore */ +} pgssLocationLen; + +/* + * Working state for computing a query jumble and producing a normalized + * query string + */ +typedef struct pgssJumbleState +{ + /* Jumble of current query tree */ + unsigned char *jumble; + + /* Number of bytes used in jumble[] */ + Size jumble_len; + + /* Array of locations of constants that should be removed */ + pgssLocationLen *clocations; + + /* Allocated length of clocations array */ + int clocations_buf_size; + + /* Current number of valid entries in clocations array */ + int clocations_count; + + /* highest Param id we've seen, in order to start normalization correctly */ + int highest_extern_param_id; +} pgssJumbleState; + +static void AppendJumble(pgssJumbleState *jstate, + const unsigned char *item, Size size); +static void JumbleQuery(pgssJumbleState *jstate, Query *query); +static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); +static void JumbleExpr(pgssJumbleState *jstate, Node *node); +static void RecordConstLocation(pgssJumbleState *jstate, int location); + +static StringInfo gen_normplan(const char *execution_plan); + +static bool need_replace(int token); + +void pgss_post_parse_analyze(ParseState *pstate, Query *query); + +void stat_statements_parser_init() +{ + prev_post_parse_analyze_hook = post_parse_analyze_hook; + post_parse_analyze_hook = pgss_post_parse_analyze; +} + +void stat_statements_parser_deinit() +{ + post_parse_analyze_hook = prev_post_parse_analyze_hook; +} + +/* + * AppendJumble: Append a value that is substantive in a given query to + * the current jumble. + */ +static void +AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size) +{ + unsigned char *jumble = jstate->jumble; + Size jumble_len = jstate->jumble_len; + + /* + * Whenever the jumble buffer is full, we hash the current contents and + * reset the buffer to contain just that hash value, thus relying on the + * hash to summarize everything so far. + */ + while (size > 0) + { + Size part_size; + + if (jumble_len >= JUMBLE_SIZE) + { + uint32 start_hash = hash_any(jumble, JUMBLE_SIZE); + + memcpy(jumble, &start_hash, sizeof(start_hash)); + jumble_len = sizeof(start_hash); + } + part_size = Min(size, JUMBLE_SIZE - jumble_len); + memcpy(jumble + jumble_len, item, part_size); + jumble_len += part_size; + item += part_size; + size -= part_size; + } + jstate->jumble_len = jumble_len; +} + +/* + * Wrappers around AppendJumble to encapsulate details of serialization + * of individual local variable elements. + */ +#define APP_JUMB(item) \ + AppendJumble(jstate, (const unsigned char *)&(item), sizeof(item)) +#define APP_JUMB_STRING(str) \ + AppendJumble(jstate, (const unsigned char *)(str), strlen(str) + 1) + +/* + * JumbleQuery: Selectively serialize the query tree, appending significant + * data to the "query jumble" while ignoring nonsignificant data. + * + * Rule of thumb for what to include is that we should ignore anything not + * semantically significant (such as alias names) as well as anything that can + * be deduced from child nodes (else we'd just be double-hashing that piece + * of information). + */ +void JumbleQuery(pgssJumbleState *jstate, Query *query) +{ + Assert(IsA(query, Query)); + Assert(query->utilityStmt == NULL); + + APP_JUMB(query->commandType); + /* resultRelation is usually predictable from commandType */ + JumbleExpr(jstate, (Node *)query->cteList); + JumbleRangeTable(jstate, query->rtable); + JumbleExpr(jstate, (Node *)query->jointree); + JumbleExpr(jstate, (Node *)query->targetList); + JumbleExpr(jstate, (Node *)query->returningList); + JumbleExpr(jstate, (Node *)query->groupClause); + JumbleExpr(jstate, query->havingQual); + JumbleExpr(jstate, (Node *)query->windowClause); + JumbleExpr(jstate, (Node *)query->distinctClause); + JumbleExpr(jstate, (Node *)query->sortClause); + JumbleExpr(jstate, query->limitOffset); + JumbleExpr(jstate, query->limitCount); + /* we ignore rowMarks */ + JumbleExpr(jstate, query->setOperations); +} + +/* + * Jumble a range table + */ +static void +JumbleRangeTable(pgssJumbleState *jstate, List *rtable) +{ + ListCell *lc; + + foreach (lc, rtable) + { + RangeTblEntry *rte = (RangeTblEntry *)lfirst(lc); + + Assert(IsA(rte, RangeTblEntry)); + APP_JUMB(rte->rtekind); + switch (rte->rtekind) + { + case RTE_RELATION: + APP_JUMB(rte->relid); + break; + case RTE_SUBQUERY: + JumbleQuery(jstate, rte->subquery); + break; + case RTE_JOIN: + APP_JUMB(rte->jointype); + break; + case RTE_FUNCTION: + JumbleExpr(jstate, (Node *)rte->functions); + break; + case RTE_VALUES: + JumbleExpr(jstate, (Node *)rte->values_lists); + break; + case RTE_CTE: + + /* + * Depending on the CTE name here isn't ideal, but it's the + * only info we have to identify the referenced WITH item. + */ + APP_JUMB_STRING(rte->ctename); + APP_JUMB(rte->ctelevelsup); + break; + default: + elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); + break; + } + } +} + +/* + * Jumble an expression tree + * + * In general this function should handle all the same node types that + * expression_tree_walker() does, and therefore it's coded to be as parallel + * to that function as possible. However, since we are only invoked on + * queries immediately post-parse-analysis, we need not handle node types + * that only appear in planning. + * + * Note: the reason we don't simply use expression_tree_walker() is that the + * point of that function is to support tree walkers that don't care about + * most tree node types, but here we care about all types. We should complain + * about any unrecognized node type. + */ +static void +JumbleExpr(pgssJumbleState *jstate, Node *node) +{ + ListCell *temp; + + if (node == NULL) + return; + + /* Guard against stack overflow due to overly complex expressions */ + check_stack_depth(); + + /* + * We always emit the node's NodeTag, then any additional fields that are + * considered significant, and then we recurse to any child nodes. + */ + APP_JUMB(node->type); + + switch (nodeTag(node)) + { + case T_Var: + { + Var *var = (Var *)node; + + APP_JUMB(var->varno); + APP_JUMB(var->varattno); + APP_JUMB(var->varlevelsup); + } + break; + case T_Const: + { + Const *c = (Const *)node; + + /* We jumble only the constant's type, not its value */ + APP_JUMB(c->consttype); + /* Also, record its parse location for query normalization */ + RecordConstLocation(jstate, c->location); + } + break; + case T_Param: + { + Param *p = (Param *)node; + + APP_JUMB(p->paramkind); + APP_JUMB(p->paramid); + APP_JUMB(p->paramtype); + } + break; + case T_Aggref: + { + Aggref *expr = (Aggref *)node; + + APP_JUMB(expr->aggfnoid); + JumbleExpr(jstate, (Node *)expr->aggdirectargs); + JumbleExpr(jstate, (Node *)expr->args); + JumbleExpr(jstate, (Node *)expr->aggorder); + JumbleExpr(jstate, (Node *)expr->aggdistinct); + JumbleExpr(jstate, (Node *)expr->aggfilter); + } + break; + case T_WindowFunc: + { + WindowFunc *expr = (WindowFunc *)node; + + APP_JUMB(expr->winfnoid); + APP_JUMB(expr->winref); + JumbleExpr(jstate, (Node *)expr->args); + JumbleExpr(jstate, (Node *)expr->aggfilter); + } + break; + case T_ArrayRef: + { + ArrayRef *aref = (ArrayRef *)node; + + JumbleExpr(jstate, (Node *)aref->refupperindexpr); + JumbleExpr(jstate, (Node *)aref->reflowerindexpr); + JumbleExpr(jstate, (Node *)aref->refexpr); + JumbleExpr(jstate, (Node *)aref->refassgnexpr); + } + break; + case T_FuncExpr: + { + FuncExpr *expr = (FuncExpr *)node; + + APP_JUMB(expr->funcid); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_NamedArgExpr: + { + NamedArgExpr *nae = (NamedArgExpr *)node; + + APP_JUMB(nae->argnumber); + JumbleExpr(jstate, (Node *)nae->arg); + } + break; + case T_OpExpr: + case T_DistinctExpr: /* struct-equivalent to OpExpr */ + case T_NullIfExpr: /* struct-equivalent to OpExpr */ + { + OpExpr *expr = (OpExpr *)node; + + APP_JUMB(expr->opno); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_ScalarArrayOpExpr: + { + ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *)node; + + APP_JUMB(expr->opno); + APP_JUMB(expr->useOr); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_BoolExpr: + { + BoolExpr *expr = (BoolExpr *)node; + + APP_JUMB(expr->boolop); + JumbleExpr(jstate, (Node *)expr->args); + } + break; + case T_SubLink: + { + SubLink *sublink = (SubLink *)node; + + APP_JUMB(sublink->subLinkType); + JumbleExpr(jstate, (Node *)sublink->testexpr); + JumbleQuery(jstate, (Query *)sublink->subselect); + } + break; + case T_FieldSelect: + { + FieldSelect *fs = (FieldSelect *)node; + + APP_JUMB(fs->fieldnum); + JumbleExpr(jstate, (Node *)fs->arg); + } + break; + case T_FieldStore: + { + FieldStore *fstore = (FieldStore *)node; + + JumbleExpr(jstate, (Node *)fstore->arg); + JumbleExpr(jstate, (Node *)fstore->newvals); + } + break; + case T_RelabelType: + { + RelabelType *rt = (RelabelType *)node; + + APP_JUMB(rt->resulttype); + JumbleExpr(jstate, (Node *)rt->arg); + } + break; + case T_CoerceViaIO: + { + CoerceViaIO *cio = (CoerceViaIO *)node; + + APP_JUMB(cio->resulttype); + JumbleExpr(jstate, (Node *)cio->arg); + } + break; + case T_ArrayCoerceExpr: + { + ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *)node; + + APP_JUMB(acexpr->resulttype); + JumbleExpr(jstate, (Node *)acexpr->arg); + } + break; + case T_ConvertRowtypeExpr: + { + ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *)node; + + APP_JUMB(crexpr->resulttype); + JumbleExpr(jstate, (Node *)crexpr->arg); + } + break; + case T_CollateExpr: + { + CollateExpr *ce = (CollateExpr *)node; + + APP_JUMB(ce->collOid); + JumbleExpr(jstate, (Node *)ce->arg); + } + break; + case T_CaseExpr: + { + CaseExpr *caseexpr = (CaseExpr *)node; + + JumbleExpr(jstate, (Node *)caseexpr->arg); + foreach (temp, caseexpr->args) + { + CaseWhen *when = (CaseWhen *)lfirst(temp); + + Assert(IsA(when, CaseWhen)); + JumbleExpr(jstate, (Node *)when->expr); + JumbleExpr(jstate, (Node *)when->result); + } + JumbleExpr(jstate, (Node *)caseexpr->defresult); + } + break; + case T_CaseTestExpr: + { + CaseTestExpr *ct = (CaseTestExpr *)node; + + APP_JUMB(ct->typeId); + } + break; + case T_ArrayExpr: + JumbleExpr(jstate, (Node *)((ArrayExpr *)node)->elements); + break; + case T_RowExpr: + JumbleExpr(jstate, (Node *)((RowExpr *)node)->args); + break; + case T_RowCompareExpr: + { + RowCompareExpr *rcexpr = (RowCompareExpr *)node; + + APP_JUMB(rcexpr->rctype); + JumbleExpr(jstate, (Node *)rcexpr->largs); + JumbleExpr(jstate, (Node *)rcexpr->rargs); + } + break; + case T_CoalesceExpr: + JumbleExpr(jstate, (Node *)((CoalesceExpr *)node)->args); + break; + case T_MinMaxExpr: + { + MinMaxExpr *mmexpr = (MinMaxExpr *)node; + + APP_JUMB(mmexpr->op); + JumbleExpr(jstate, (Node *)mmexpr->args); + } + break; + case T_XmlExpr: + { + XmlExpr *xexpr = (XmlExpr *)node; + + APP_JUMB(xexpr->op); + JumbleExpr(jstate, (Node *)xexpr->named_args); + JumbleExpr(jstate, (Node *)xexpr->args); + } + break; + case T_NullTest: + { + NullTest *nt = (NullTest *)node; + + APP_JUMB(nt->nulltesttype); + JumbleExpr(jstate, (Node *)nt->arg); + } + break; + case T_BooleanTest: + { + BooleanTest *bt = (BooleanTest *)node; + + APP_JUMB(bt->booltesttype); + JumbleExpr(jstate, (Node *)bt->arg); + } + break; + case T_CoerceToDomain: + { + CoerceToDomain *cd = (CoerceToDomain *)node; + + APP_JUMB(cd->resulttype); + JumbleExpr(jstate, (Node *)cd->arg); + } + break; + case T_CoerceToDomainValue: + { + CoerceToDomainValue *cdv = (CoerceToDomainValue *)node; + + APP_JUMB(cdv->typeId); + } + break; + case T_SetToDefault: + { + SetToDefault *sd = (SetToDefault *)node; + + APP_JUMB(sd->typeId); + } + break; + case T_CurrentOfExpr: + { + CurrentOfExpr *ce = (CurrentOfExpr *)node; + + APP_JUMB(ce->cvarno); + if (ce->cursor_name) + APP_JUMB_STRING(ce->cursor_name); + APP_JUMB(ce->cursor_param); + } + break; + case T_TargetEntry: + { + TargetEntry *tle = (TargetEntry *)node; + + APP_JUMB(tle->resno); + APP_JUMB(tle->ressortgroupref); + JumbleExpr(jstate, (Node *)tle->expr); + } + break; + case T_RangeTblRef: + { + RangeTblRef *rtr = (RangeTblRef *)node; + + APP_JUMB(rtr->rtindex); + } + break; + case T_JoinExpr: + { + JoinExpr *join = (JoinExpr *)node; + + APP_JUMB(join->jointype); + APP_JUMB(join->isNatural); + APP_JUMB(join->rtindex); + JumbleExpr(jstate, join->larg); + JumbleExpr(jstate, join->rarg); + JumbleExpr(jstate, join->quals); + } + break; + case T_FromExpr: + { + FromExpr *from = (FromExpr *)node; + + JumbleExpr(jstate, (Node *)from->fromlist); + JumbleExpr(jstate, from->quals); + } + break; + case T_List: + foreach (temp, (List *)node) + { + JumbleExpr(jstate, (Node *)lfirst(temp)); + } + break; + case T_SortGroupClause: + { + SortGroupClause *sgc = (SortGroupClause *)node; + + APP_JUMB(sgc->tleSortGroupRef); + APP_JUMB(sgc->eqop); + APP_JUMB(sgc->sortop); + APP_JUMB(sgc->nulls_first); + } + break; + case T_WindowClause: + { + WindowClause *wc = (WindowClause *)node; + + APP_JUMB(wc->winref); + APP_JUMB(wc->frameOptions); + JumbleExpr(jstate, (Node *)wc->partitionClause); + JumbleExpr(jstate, (Node *)wc->orderClause); + JumbleExpr(jstate, wc->startOffset); + JumbleExpr(jstate, wc->endOffset); + } + break; + case T_CommonTableExpr: + { + CommonTableExpr *cte = (CommonTableExpr *)node; + + /* we store the string name because RTE_CTE RTEs need it */ + APP_JUMB_STRING(cte->ctename); + JumbleQuery(jstate, (Query *)cte->ctequery); + } + break; + case T_SetOperationStmt: + { + SetOperationStmt *setop = (SetOperationStmt *)node; + + APP_JUMB(setop->op); + APP_JUMB(setop->all); + JumbleExpr(jstate, setop->larg); + JumbleExpr(jstate, setop->rarg); + } + break; + case T_RangeTblFunction: + { + RangeTblFunction *rtfunc = (RangeTblFunction *)node; + + JumbleExpr(jstate, rtfunc->funcexpr); + } + break; + default: + /* Only a warning, since we can stumble along anyway */ + elog(WARNING, "unrecognized node type: %d", + (int)nodeTag(node)); + break; + } +} + +/* + * Record location of constant within query string of query tree + * that is currently being walked. + */ +static void +RecordConstLocation(pgssJumbleState *jstate, int location) +{ + /* -1 indicates unknown or undefined location */ + if (location >= 0) + { + /* enlarge array if needed */ + if (jstate->clocations_count >= jstate->clocations_buf_size) + { + jstate->clocations_buf_size *= 2; + jstate->clocations = (pgssLocationLen *) + repalloc(jstate->clocations, + jstate->clocations_buf_size * + sizeof(pgssLocationLen)); + } + jstate->clocations[jstate->clocations_count].location = location; + /* initialize lengths to -1 to simplify fill_in_constant_lengths */ + jstate->clocations[jstate->clocations_count].length = -1; + jstate->clocations_count++; + } +} + +/* check if token should be replaced by substitute varable */ +static bool +need_replace(int token) +{ + return (token == FCONST) || (token == ICONST) || (token == SCONST) || (token == BCONST) || (token == XCONST); +} + +/* + * gen_normplan - parse execution plan using flex and replace all CONST to + * substitute variables. + */ +static StringInfo +gen_normplan(const char *execution_plan) +{ + core_yyscan_t yyscanner; + core_yy_extra_type yyextra; + core_YYSTYPE yylval; + YYLTYPE yylloc; + int tok; + int bind_prefix = 1; + char *tmp_str; + YYLTYPE last_yylloc = 0; + int last_tok = 0; + StringInfo plan_out = makeStringInfo(); + ; + + yyscanner = scanner_init(execution_plan, + &yyextra, +#if PG_VERSION_NUM >= 120000 + &ScanKeywords, + ScanKeywordTokens +#else + ScanKeywords, + NumScanKeywords +#endif + ); + + for (;;) + { + /* get the next lexem */ + tok = core_yylex(&yylval, &yylloc, yyscanner); + + /* now we store end previsous lexem in yylloc - so could prcess it */ + if (need_replace(last_tok)) + { + /* substitute variable instead of CONST */ + int s_len = asprintf(&tmp_str, "$%i", bind_prefix++); + if (s_len > 0) + { + appendStringInfoString(plan_out, tmp_str); + free(tmp_str); + } + else + { + appendStringInfoString(plan_out, "??"); + } + } + else + { + /* do not change - just copy as-is */ + tmp_str = strndup((char *)execution_plan + last_yylloc, yylloc - last_yylloc); + appendStringInfoString(plan_out, tmp_str); + free(tmp_str); + } + /* check if further parsing not needed */ + if (tok == 0) + break; + last_tok = tok; + last_yylloc = yylloc; + } + + scanner_finish(yyscanner); + + return plan_out; +} + +uint64_t get_plan_id(QueryDesc *queryDesc) +{ + if (!queryDesc->sourceText) + return 0; + StringInfo normalized = gen_normplan(queryDesc->sourceText); + return hash_any((unsigned char *)normalized->data, normalized->len); +} + +/* + * Post-parse-analysis hook: mark query with a queryId + */ +void pgss_post_parse_analyze(ParseState *pstate, Query *query) +{ + pgssJumbleState jstate; + + if (prev_post_parse_analyze_hook) + prev_post_parse_analyze_hook(pstate, query); + + /* Assert we didn't do this already */ + Assert(query->queryId == 0); + + /* + * Utility statements get queryId zero. We do this even in cases where + * the statement contains an optimizable statement for which a queryId + * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, + * runtime control will first go through ProcessUtility and then the + * executor, and we don't want the executor hooks to do anything, since we + * are already measuring the statement's costs at the utility level. + */ + if (query->utilityStmt) + { + query->queryId = 0; + return; + } + + /* Set up workspace for query jumbling */ + jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble_len = 0; + jstate.clocations_buf_size = 32; + jstate.clocations = (pgssLocationLen *) + palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations_count = 0; + + /* Compute query ID and mark the Query node with it */ + JumbleQuery(&jstate, query); + query->queryId = hash_any(jstate.jumble, jstate.jumble_len); + + /* + * If we are unlucky enough to get a hash of zero, use 1 instead, to + * prevent confusion with the utility-statement case. + */ + if (query->queryId == 0) + query->queryId = 1; +} \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h new file mode 100644 index 00000000000..274f96aebaf --- /dev/null +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -0,0 +1,15 @@ +#pragma once + +#ifdef __cplusplus +extern "C" +{ +#endif + +extern void stat_statements_parser_init(void); +extern void stat_statements_parser_deinit(void); + +#ifdef __cplusplus +} +#endif + +uint64_t get_plan_id(QueryDesc *queryDesc); \ No newline at end of file diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c new file mode 100644 index 00000000000..69475ea5079 --- /dev/null +++ b/src/yagp_hooks_collector.c @@ -0,0 +1,22 @@ +#include "postgres.h" +#include "cdb/cdbvars.h" +#include "fmgr.h" + +#include "hook_wrappers.h" + +PG_MODULE_MAGIC; + +void _PG_init(void); +void _PG_fini(void); + +void _PG_init(void) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + hooks_init(); + } +} + +void _PG_fini(void) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + hooks_deinit(); + } +} diff --git a/yagp-hooks-collector.control b/yagp-hooks-collector.control new file mode 100644 index 00000000000..82c189a88fc --- /dev/null +++ b/yagp-hooks-collector.control @@ -0,0 +1,5 @@ +# yagp-hooks-collector extension +comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' +default_version = '1.0' +module_pathname = '$libdir/yagp-hooks-collector' +superuser = true From 9a726e3a8cdd372b1ce1ab6e96ca31f0dcbc7d9b Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 Mar 2023 17:07:23 +0300 Subject: [PATCH 059/167] [yagp_hooks_collector] Fix segfault in plan text generator Guard against NULL plan state when generating EXPLAIN output. --- src/EventSender.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index bb4765adeb1..b1815a22bf8 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -130,10 +130,13 @@ void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plantext(), queryDesc); - qi->set_plan_id(get_plan_id(queryDesc)); - qi->set_query_id(queryDesc->plannedstmt->queryId); + if (queryDesc->planstate) + { + set_plan_text(qi->mutable_plantext(), queryDesc); + qi->set_plan_id(get_plan_id(queryDesc)); + } } + qi->set_query_id(queryDesc->plannedstmt->queryId); } } // namespace From 451ded570e41cd233216f64ec588a641bf1b6d55 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 29 Mar 2023 16:10:20 +0300 Subject: [PATCH 060/167] [yagp_hooks_collector] Add executor instrumentation, /proc stats, and normalized texts Collect spill info (file count, bytes written). Generate normalized query and plan texts using a pg_stat_statements-derived parser. Collect buffer I/O counters, tuple counts, timing, and /proc/self CPU/memory/IO statistics. --- Makefile | 1 - protos/yagpcc_metrics.proto | 42 +-- protos/yagpcc_set_service.proto | 20 +- src/EventSender.cpp | 164 +++++++----- src/EventSender.h | 4 +- src/ProcStats.cpp | 119 +++++++++ src/ProcStats.h | 7 + src/SpillInfoWrapper.c | 21 ++ src/hook_wrappers.cpp | 22 +- .../pg_stat_statements_ya_parser.c | 248 +++++++++++++++++- .../pg_stat_statements_ya_parser.h | 3 +- 11 files changed, 519 insertions(+), 132 deletions(-) create mode 100644 src/ProcStats.cpp create mode 100644 src/ProcStats.h create mode 100644 src/SpillInfoWrapper.c diff --git a/Makefile b/Makefile index 15c5dabb70e..0a21cf136ff 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. -all: all check install installdirs installcheck installcheck-parallel uninstall clean distclean maintainer-clean dist distcheck world check-world install-world installcheck-world installcheck-resgroup installcheck-resgroup-v2: @if [ ! -f GNUmakefile ] ; then \ diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index b7e255484c7..f00f329a208 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -27,9 +27,12 @@ message QueryInfo { PlanGenerator generator = 1; uint64 query_id = 2; uint64 plan_id = 3; - string queryText = 4; - string planText = 5; - SessionInfo sessionInfo = 6; + string query_text = 4; + string plan_text = 5; + string temlate_query_text = 6; + string temlate_plan_text = 7; + string userName = 8; + string databaseName = 9; } enum PlanGenerator @@ -45,40 +48,17 @@ message GPMetrics { SpillInfo spill = 3; } -message QueryInfoHeader { - int32 pid = 1; - GpId gpIdentity = 2; - - int32 tmid = 3; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ - int32 ssid = 4; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ - int32 ccnt = 5; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ - int32 sliceid = 6; /* slice identificator, 0 means general info for the whole query */ +message QueryKey { + int32 tmid = 1; /* A time identifier for a particular query. All records associated with the query will have the same tmid. */ + int32 ssid = 2; /* The session id as shown by gp_session_id. All records associated with the query will have the same ssid */ + int32 ccnt = 3; /* The command number within this session as shown by gp_command_count. All records associated with the query will have the same ccnt */ } -message GpId { +message SegmentKey { int32 dbid = 1; /* the dbid of this database */ int32 segindex = 2; /* content indicator: -1 for entry database, * 0, ..., n-1 for segment database * * a primary and its mirror have the same segIndex */ - GpRole gp_role = 3; - GpRole gp_session_role = 4; -} - -enum GpRole -{ - GP_ROLE_UNSPECIFIED = 0; - GP_ROLE_UTILITY = 1; /* Operating as a simple database engine */ - GP_ROLE_DISPATCH = 2; /* Operating as the parallel query dispatcher */ - GP_ROLE_EXECUTE = 3; /* Operating as a parallel query executor */ - GP_ROLE_UNDEFINED = 4; /* Should never see this role in use */ -} - -message SessionInfo { - string sql = 1; - string userName = 2; - string databaseName = 3; - string resourceGroup = 4; - string applicationName = 5; } message SystemStat { diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 0bef72891ee..97c5691a6f5 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -27,19 +27,19 @@ enum MetricResponseStatusCode { } message SetQueryReq { - QueryStatus query_status = 1; + QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; - - QueryInfoHeader header = 3; - QueryInfo query_info = 4; - GPMetrics query_metrics = 5; - repeated MetricPlan plan_tree = 6; + QueryKey query_key = 3; + QueryInfo query_info = 4; + GPMetrics query_metrics = 5; + repeated MetricPlan plan_tree = 6; } message SetPlanNodeReq { - PlanNodeStatus node_status = 1; + PlanNodeStatus node_status = 1; google.protobuf.Timestamp datetime = 2; - QueryInfoHeader header = 3; - GPMetrics node_metrics = 4; - MetricPlan plan_node = 5; + QueryKey query_key = 3; + SegmentKey segment_key = 4; + GPMetrics node_metrics = 5; + MetricPlan plan_node = 6; } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b1815a22bf8..d8145b811a4 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,11 +1,13 @@ #include "EventSender.h" #include "GrpcConnector.h" +#include "ProcStats.h" #include "protos/yagpcc_set_service.pb.h" #include extern "C" { #include "postgres.h" +#include "access/hash.h" #include "utils/metrics_utils.h" #include "utils/elog.h" #include "executor/executor.h" @@ -18,10 +20,13 @@ extern "C" #include "tcop/utility.h" #include "pg_stat_statements_ya_parser.h" + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); } namespace { + std::string* get_user_name() { const char *username = GetConfigOption("session_authorization", false, false); @@ -36,26 +41,6 @@ std::string* get_db_name() return result; } -std::string* get_rg_name() -{ - auto userId = GetUserId(); - if (!OidIsValid(userId)) - return nullptr; - auto groupId = GetResGroupIdForRole(userId); - if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); - if (rgname == nullptr) - return nullptr; - pfree(rgname); - return new std::string(rgname); -} - -std::string* get_app_name() -{ - return application_name ? new std::string(application_name) : nullptr; -} - int get_cur_slice_id(QueryDesc *desc) { if (!desc->estate) @@ -75,33 +60,22 @@ google::protobuf::Timestamp current_ts() return current_ts; } -void set_header(yagpcc::QueryInfoHeader *header, QueryDesc *queryDesc) +void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { - header->set_pid(MyProcPid); - auto gpId = header->mutable_gpidentity(); - gpId->set_dbid(GpIdentity.dbid); - gpId->set_segindex(GpIdentity.segindex); - gpId->set_gp_role(static_cast(Gp_role)); - gpId->set_gp_session_role(static_cast(Gp_session_role)); - header->set_ssid(gp_session_id); - header->set_ccnt(gp_command_count); - header->set_sliceid(get_cur_slice_id(queryDesc)); + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); int32 tmid = 0; gpmon_gettmid(&tmid); - header->set_tmid(tmid); + key->set_tmid(tmid); } -void set_session_info(yagpcc::SessionInfo *si, QueryDesc *queryDesc) +void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { - if (queryDesc->sourceText) - *si->mutable_sql() = std::string(queryDesc->sourceText); - si->set_allocated_applicationname(get_app_name()); - si->set_allocated_databasename(get_db_name()); - si->set_allocated_resourcegroup(get_rg_name()); - si->set_allocated_username(get_user_name()); + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); } -ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { ExplainState es; ExplainInitState(&es); @@ -109,74 +83,130 @@ ExplainState get_explain_state(QueryDesc *queryDesc, bool costs) es.verbose = true; es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); - ExplainPrintPlan(&es, queryDesc); + ExplainPrintPlan(&es, query_desc); ExplainEndOutput(&es); return es; } -void set_plan_text(std::string *plan_text, QueryDesc *queryDesc) +void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { - auto es = get_explain_state(queryDesc, true); + auto es = get_explain_state(query_desc, true); *plan_text = std::string(es.str->data, es.str->len); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *queryDesc) +void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - set_session_info(qi->mutable_sessioninfo(), queryDesc); - if (queryDesc->sourceText) - *qi->mutable_querytext() = queryDesc->sourceText; - if (queryDesc->plannedstmt) - { - qi->set_generator(queryDesc->plannedstmt->planGen == PLANGEN_OPTIMIZER + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - if (queryDesc->planstate) - { - set_plan_text(qi->mutable_plantext(), queryDesc); - qi->set_plan_id(get_plan_id(queryDesc)); - } + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + //TODO: free stringinfo? +} + +void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) +{ + *qi->mutable_query_text() = query_desc->sourceText; + char* norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_temlate_query_text() = std::string(norm_query); + pfree(norm_query); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) +{ + if (query_desc->sourceText) + set_query_text(qi, query_desc); + if (query_desc->plannedstmt) + { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); } - qi->set_query_id(queryDesc->plannedstmt->queryId); + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) +{ + auto instrument = query_desc->planstate->instrument; + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); } + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) +{ + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + if (query_desc->planstate->instrument) + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + fill_self_stats(metrics->mutable_systemstat()); +} + + } // namespace -void EventSender::ExecutorStart(QueryDesc *queryDesc, int /* eflags*/) +void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { - elog(DEBUG1, "Query %s start recording", queryDesc->sourceText); + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + + elog(DEBUG1, "Query %s start recording", query_desc->sourceText); yagpcc::SetQueryReq req; req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); *req.mutable_datetime() = current_ts(); - set_header(req.mutable_header(), queryDesc); - set_query_info(req.mutable_query_info(), queryDesc); + set_query_key(req.mutable_query_key(), query_desc); auto result = connector->set_metric_query(req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { elog(WARNING, "Query %s start reporting failed with an error %s", - queryDesc->sourceText, result.error_text().c_str()); + query_desc->sourceText, result.error_text().c_str()); } else { - elog(DEBUG1, "Query %s start successful", queryDesc->sourceText); + elog(DEBUG1, "Query %s start successful", query_desc->sourceText); } } -void EventSender::ExecutorFinish(QueryDesc *queryDesc) +void EventSender::ExecutorFinish(QueryDesc *query_desc) { - elog(DEBUG1, "Query %s finish recording", queryDesc->sourceText); + elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); yagpcc::SetQueryReq req; req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); *req.mutable_datetime() = current_ts(); - set_header(req.mutable_header(), queryDesc); - set_query_info(req.mutable_query_info(), queryDesc); + set_query_key(req.mutable_query_key(), query_desc); + set_query_info(req.mutable_query_info(), query_desc); + set_gp_metrics(req.mutable_query_metrics(), query_desc); auto result = connector->set_metric_query(req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { elog(WARNING, "Query %s finish reporting failed with an error %s", - queryDesc->sourceText, result.error_text().c_str()); + query_desc->sourceText, result.error_text().c_str()); } else { - elog(DEBUG1, "Query %s finish successful", queryDesc->sourceText); + elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); } } diff --git a/src/EventSender.h b/src/EventSender.h index 70868f6c757..bd02455ca7e 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -9,8 +9,8 @@ struct QueryDesc; class EventSender { public: - void ExecutorStart(QueryDesc *queryDesc, int eflags); - void ExecutorFinish(QueryDesc *queryDesc); + void ExecutorStart(QueryDesc *query_desc, int eflags); + void ExecutorFinish(QueryDesc *query_desc); static EventSender *instance(); private: diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp new file mode 100644 index 00000000000..34c5d05719e --- /dev/null +++ b/src/ProcStats.cpp @@ -0,0 +1,119 @@ +#include "ProcStats.h" +#include "yagpcc_metrics.pb.h" +#include +#include +#include + +extern "C" +{ +#include "postgres.h" +#include "utils/elog.h" +} + +namespace { +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name); + +void fill_io_stats(yagpcc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); +} + +void fill_cpu_stats(yagpcc::SystemStat *stats) +{ + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int STARTTIME_ID = 21; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); + + double uptime; + { + std::ifstream proc_stat("/proc/uptime"); + proc_stat >> uptime; + } + + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + double start_time = 0; + for (int i = 0; i <= RSS_ID; ++i) + { + switch (i) + { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps); + break; + case STARTTIME_ID: + uint64_t starttime; + proc_stat >> starttime; + start_time = static_cast(starttime) / tps; + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; + } + stats->set_runningtimeseconds(uptime - start_time); + } +} + +void fill_status_stats(yagpcc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) + { + if (key == "VmPeak:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") + elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); + } + else if (key == "VmSize:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") + elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); + } + } +} +} // namespace + +void fill_self_stats(yagpcc::SystemStat *stats) +{ + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); +} \ No newline at end of file diff --git a/src/ProcStats.h b/src/ProcStats.h new file mode 100644 index 00000000000..30a90a60519 --- /dev/null +++ b/src/ProcStats.h @@ -0,0 +1,7 @@ +#pragma once + +namespace yagpcc { +class SystemStat; +} + +void fill_self_stats(yagpcc::SystemStat *stats); \ No newline at end of file diff --git a/src/SpillInfoWrapper.c b/src/SpillInfoWrapper.c new file mode 100644 index 00000000000..c6ace0a693f --- /dev/null +++ b/src/SpillInfoWrapper.c @@ -0,0 +1,21 @@ +#include "postgres.h" +#include "utils/workfile_mgr.h" + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); + +void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes) +{ + int count = 0; + int i = 0; + workfile_set *workfiles = workfile_mgr_cache_entries_get_copy(&count); + workfile_set *wf_iter = workfiles; + for (i = 0; i < count; ++i, ++wf_iter) + { + if (wf_iter->active && wf_iter->session_id == ssid && wf_iter->command_count == ccid) + { + *file_count += wf_iter->num_files; + *total_bytes += wf_iter->total_bytes; + } + } + pfree(workfiles); +} \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 9f3200c006f..1dabb59ab3f 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -19,8 +19,8 @@ extern "C" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; -static void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags); -static void ya_ExecutorFinish_hook(QueryDesc *queryDesc); +static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorFinish_hook(QueryDesc *query_desc); #define REPLACE_HOOK(hookName) \ previous_##hookName = hookName; \ @@ -56,12 +56,22 @@ void hooks_deinit() else \ standard_##hookName(__VA_ARGS__); -void ya_ExecutorStart_hook(QueryDesc *queryDesc, int eflags) +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - CREATE_HOOK_WRAPPER(ExecutorStart, queryDesc, eflags); + CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); + PG_TRY(); + { + EventSender::instance()->ExecutorStart(query_desc, eflags); + } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + PG_RE_THROW(); + } + PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *queryDesc) +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - CREATE_HOOK_WRAPPER(ExecutorFinish, queryDesc); + CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index f14742337bd..ae79e7dc40a 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -1,3 +1,6 @@ +// NOTE: this file is just a bunch of code borrowed from pg_stat_statements for PG 9.4 +// and from our own inhouse implementation of pg_stat_statements for managed PG + #include "postgres.h" #include @@ -67,14 +70,15 @@ static void JumbleQuery(pgssJumbleState *jstate, Query *query); static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); static void JumbleExpr(pgssJumbleState *jstate, Node *node); static void RecordConstLocation(pgssJumbleState *jstate, int location); - -static StringInfo gen_normplan(const char *execution_plan); - +static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query); +static int comp_location(const void *a, const void *b); +StringInfo gen_normplan(const char *execution_plan); static bool need_replace(int token); - void pgss_post_parse_analyze(ParseState *pstate, Query *query); +static char *generate_normalized_query(pgssJumbleState *jstate, const char *query, + int *query_len_p, int encoding); -void stat_statements_parser_init() + void stat_statements_parser_init() { prev_post_parse_analyze_hook = post_parse_analyze_hook; post_parse_analyze_hook = pgss_post_parse_analyze; @@ -650,7 +654,7 @@ need_replace(int token) * gen_normplan - parse execution plan using flex and replace all CONST to * substitute variables. */ -static StringInfo +StringInfo gen_normplan(const char *execution_plan) { core_yyscan_t yyscanner; @@ -715,14 +719,6 @@ gen_normplan(const char *execution_plan) return plan_out; } -uint64_t get_plan_id(QueryDesc *queryDesc) -{ - if (!queryDesc->sourceText) - return 0; - StringInfo normalized = gen_normplan(queryDesc->sourceText); - return hash_any((unsigned char *)normalized->data, normalized->len); -} - /* * Post-parse-analysis hook: mark query with a queryId */ @@ -768,4 +764,228 @@ void pgss_post_parse_analyze(ParseState *pstate, Query *query) */ if (query->queryId == 0) query->queryId = 1; +} + +/* + * comp_location: comparator for qsorting pgssLocationLen structs by location + */ +static int +comp_location(const void *a, const void *b) +{ + int l = ((const pgssLocationLen *) a)->location; + int r = ((const pgssLocationLen *) b)->location; + + if (l < r) + return -1; + else if (l > r) + return +1; + else + return 0; +} + +/* + * Given a valid SQL string and an array of constant-location records, + * fill in the textual lengths of those constants. + * + * The constants may use any allowed constant syntax, such as float literals, + * bit-strings, single-quoted strings and dollar-quoted strings. This is + * accomplished by using the public API for the core scanner. + * + * It is the caller's job to ensure that the string is a valid SQL statement + * with constants at the indicated locations. Since in practice the string + * has already been parsed, and the locations that the caller provides will + * have originated from within the authoritative parser, this should not be + * a problem. + * + * Duplicate constant pointers are possible, and will have their lengths + * marked as '-1', so that they are later ignored. (Actually, we assume the + * lengths were initialized as -1 to start with, and don't change them here.) + * + * N.B. There is an assumption that a '-' character at a Const location begins + * a negative numeric constant. This precludes there ever being another + * reason for a constant to start with a '-'. + */ +static void +fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) +{ + pgssLocationLen *locs; + core_yyscan_t yyscanner; + core_yy_extra_type yyextra; + core_YYSTYPE yylval; + YYLTYPE yylloc; + int last_loc = -1; + int i; + + /* + * Sort the records by location so that we can process them in order while + * scanning the query text. + */ + if (jstate->clocations_count > 1) + qsort(jstate->clocations, jstate->clocations_count, + sizeof(pgssLocationLen), comp_location); + locs = jstate->clocations; + + /* initialize the flex scanner --- should match raw_parser() */ + yyscanner = scanner_init(query, + &yyextra, + ScanKeywords, + NumScanKeywords); + + /* Search for each constant, in sequence */ + for (i = 0; i < jstate->clocations_count; i++) + { + int loc = locs[i].location; + int tok; + + Assert(loc >= 0); + + if (loc <= last_loc) + continue; /* Duplicate constant, ignore */ + + /* Lex tokens until we find the desired constant */ + for (;;) + { + tok = core_yylex(&yylval, &yylloc, yyscanner); + + /* We should not hit end-of-string, but if we do, behave sanely */ + if (tok == 0) + break; /* out of inner for-loop */ + + /* + * We should find the token position exactly, but if we somehow + * run past it, work with that. + */ + if (yylloc >= loc) + { + if (query[loc] == '-') + { + /* + * It's a negative value - this is the one and only case + * where we replace more than a single token. + * + * Do not compensate for the core system's special-case + * adjustment of location to that of the leading '-' + * operator in the event of a negative constant. It is + * also useful for our purposes to start from the minus + * symbol. In this way, queries like "select * from foo + * where bar = 1" and "select * from foo where bar = -2" + * will have identical normalized query strings. + */ + tok = core_yylex(&yylval, &yylloc, yyscanner); + if (tok == 0) + break; /* out of inner for-loop */ + } + + /* + * We now rely on the assumption that flex has placed a zero + * byte after the text of the current token in scanbuf. + */ + locs[i].length = strlen(yyextra.scanbuf + loc); + break; /* out of inner for-loop */ + } + } + + /* If we hit end-of-string, give up, leaving remaining lengths -1 */ + if (tok == 0) + break; + + last_loc = loc; + } + + scanner_finish(yyscanner); +} + +/* + * Generate a normalized version of the query string that will be used to + * represent all similar queries. + * + * Note that the normalized representation may well vary depending on + * just which "equivalent" query is used to create the hashtable entry. + * We assume this is OK. + * + * *query_len_p contains the input string length, and is updated with + * the result string length (which cannot be longer) on exit. + * + * Returns a palloc'd string. + */ +static char * +generate_normalized_query(pgssJumbleState *jstate, const char *query, + int *query_len_p, int encoding) +{ + char *norm_query; + int query_len = *query_len_p; + int i, + len_to_wrt, /* Length (in bytes) to write */ + quer_loc = 0, /* Source query byte location */ + n_quer_loc = 0, /* Normalized query byte location */ + last_off = 0, /* Offset from start for previous tok */ + last_tok_len = 0; /* Length (in bytes) of that tok */ + + /* + * Get constants' lengths (core system only gives us locations). Note + * this also ensures the items are sorted by location. + */ + fill_in_constant_lengths(jstate, query); + + /* Allocate result buffer */ + norm_query = palloc(query_len + 1); + + for (i = 0; i < jstate->clocations_count; i++) + { + int off, /* Offset from start for cur tok */ + tok_len; /* Length (in bytes) of that tok */ + + off = jstate->clocations[i].location; + tok_len = jstate->clocations[i].length; + + if (tok_len < 0) + continue; /* ignore any duplicates */ + + /* Copy next chunk (what precedes the next constant) */ + len_to_wrt = off - last_off; + len_to_wrt -= last_tok_len; + + Assert(len_to_wrt >= 0); + memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); + n_quer_loc += len_to_wrt; + + /* And insert a '?' in place of the constant token */ + norm_query[n_quer_loc++] = '?'; + + quer_loc = off + tok_len; + last_off = off; + last_tok_len = tok_len; + } + + /* + * We've copied up until the last ignorable constant. Copy over the + * remaining bytes of the original query string. + */ + len_to_wrt = query_len - quer_loc; + + Assert(len_to_wrt >= 0); + memcpy(norm_query + n_quer_loc, query + quer_loc, len_to_wrt); + n_quer_loc += len_to_wrt; + + Assert(n_quer_loc <= query_len); + norm_query[n_quer_loc] = '\0'; + + *query_len_p = n_quer_loc; + return norm_query; +} + +char *gen_normquery(const char *query) +{ + if (!query) { + return NULL; + } + pgssJumbleState jstate; + jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble_len = 0; + jstate.clocations_buf_size = 32; + jstate.clocations = (pgssLocationLen *) + palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations_count = 0; + int query_len = strlen(query); + return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index 274f96aebaf..aa9cd217e31 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -12,4 +12,5 @@ extern void stat_statements_parser_deinit(void); } #endif -uint64_t get_plan_id(QueryDesc *queryDesc); \ No newline at end of file +StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query); \ No newline at end of file From 6874e12b266a2eac759ec6ff12d26459dd8f86a8 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 6 Apr 2023 13:24:25 +0300 Subject: [PATCH 061/167] [yagp_hooks_collector] Apply llvm code style --- Makefile | 1 - src/EventSender.cpp | 367 ++++++++++++++++++++---------------------- src/EventSender.h | 13 +- src/GrpcConnector.cpp | 68 ++++---- src/GrpcConnector.h | 13 +- src/ProcStats.cpp | 183 ++++++++++----------- src/hook_wrappers.cpp | 83 +++++----- 7 files changed, 338 insertions(+), 390 deletions(-) diff --git a/Makefile b/Makefile index 0a21cf136ff..91be52c4468 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. - # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. diff --git a/src/EventSender.cpp b/src/EventSender.cpp index d8145b811a4..b7c3cd70b85 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -4,8 +4,7 @@ #include "protos/yagpcc_set_service.pb.h" #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "access/hash.h" #include "utils/metrics_utils.h" @@ -21,202 +20,178 @@ extern "C" #include "tcop/utility.h" #include "pg_stat_statements_ya_parser.h" -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); -} - -namespace -{ - -std::string* get_user_name() -{ - const char *username = GetConfigOption("session_authorization", false, false); - return username ? new std::string(username) : nullptr; -} - -std::string* get_db_name() -{ - char *dbname = get_database_name(MyDatabaseId); - std::string* result = dbname ? new std::string(dbname) : nullptr; - pfree(dbname); - return result; -} - -int get_cur_slice_id(QueryDesc *desc) -{ - if (!desc->estate) - { - return 0; - } - return LocallyExecutingSliceIndex(desc->estate); -} - -google::protobuf::Timestamp current_ts() -{ - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; -} - -void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) -{ - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gpmon_gettmid(&tmid); - key->set_tmid(tmid); -} - -void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) -{ - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); -} - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) -{ - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; -} - -void set_plan_text(std::string *plan_text, QueryDesc *query_desc) -{ - auto es = get_explain_state(query_desc, true); - *plan_text = std::string(es.str->data, es.str->len); -} - -void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - //TODO: free stringinfo? -} - -void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - *qi->mutable_query_text() = query_desc->sourceText; - char* norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_temlate_query_text() = std::string(norm_query); - pfree(norm_query); -} - -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) -{ - if (query_desc->sourceText) - set_query_text(qi, query_desc); - if (query_desc->plannedstmt) - { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); -} - -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) -{ - auto instrument = query_desc->planstate->instrument; - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) -{ - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - if (query_desc->planstate->instrument) - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); - fill_self_stats(metrics->mutable_systemstat()); +void get_spill_info(int ssid, int ccid, int32_t *file_count, + int64_t *total_bytes); } +namespace { + +std::string *get_user_name() { + const char *username = GetConfigOption("session_authorization", false, false); + return username ? new std::string(username) : nullptr; +} + +std::string *get_db_name() { + char *dbname = get_database_name(MyDatabaseId); + std::string *result = dbname ? new std::string(dbname) : nullptr; + pfree(dbname); + return result; +} + +int get_cur_slice_id(QueryDesc *desc) { + if (!desc->estate) { + return 0; + } + return LocallyExecutingSliceIndex(desc->estate); +} + +google::protobuf::Timestamp current_ts() { + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gpmon_gettmid(&tmid); + key->set_tmid(tmid); +} + +void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); +} + +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; +} + +void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { + auto es = get_explain_state(query_desc, true); + *plan_text = std::string(es.str->data, es.str->len); +} + +void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + // TODO: free stringinfo? +} + +void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + *qi->mutable_query_text() = query_desc->sourceText; + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_temlate_query_text() = std::string(norm_query); + pfree(norm_query); +} + +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { + if (query_desc->sourceText) { + set_query_text(qi, query_desc); + } + if (query_desc->plannedstmt) { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); + } + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, + QueryDesc *query_desc) { + auto instrument = query_desc->planstate->instrument; + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, + &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + if (query_desc->planstate->instrument) { + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + } + fill_self_stats(metrics->mutable_systemstat()); +} } // namespace -void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) -{ - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - - elog(DEBUG1, "Query %s start recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) - { - elog(WARNING, "Query %s start reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } - else - { - elog(DEBUG1, "Query %s start successful", query_desc->sourceText); - } -} - -void EventSender::ExecutorFinish(QueryDesc *query_desc) -{ - elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_query_info(req.mutable_query_info(), query_desc); - set_gp_metrics(req.mutable_query_metrics(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) - { - elog(WARNING, "Query %s finish reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } - else - { - elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); - } -} - -EventSender *EventSender::instance() -{ - static EventSender sender; - return &sender; -} - -EventSender::EventSender() -{ - connector = std::make_unique(); -} \ No newline at end of file +void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + + elog(DEBUG1, "Query %s start recording", query_desc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { + elog(WARNING, "Query %s start reporting failed with an error %s", + query_desc->sourceText, result.error_text().c_str()); + } else { + elog(DEBUG1, "Query %s start successful", query_desc->sourceText); + } +} + +void EventSender::ExecutorFinish(QueryDesc *query_desc) { + elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); + yagpcc::SetQueryReq req; + req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + set_query_info(req.mutable_query_info(), query_desc); + set_gp_metrics(req.mutable_query_metrics(), query_desc); + auto result = connector->set_metric_query(req); + if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { + elog(WARNING, "Query %s finish reporting failed with an error %s", + query_desc->sourceText, result.error_text().c_str()); + } else { + elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); + } +} + +EventSender *EventSender::instance() { + static EventSender sender; + return &sender; +} + +EventSender::EventSender() { connector = std::make_unique(); } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index bd02455ca7e..d69958db9b0 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -6,14 +6,13 @@ class GrpcConnector; struct QueryDesc; -class EventSender -{ +class EventSender { public: - void ExecutorStart(QueryDesc *query_desc, int eflags); - void ExecutorFinish(QueryDesc *query_desc); - static EventSender *instance(); + void ExecutorStart(QueryDesc *query_desc, int eflags); + void ExecutorFinish(QueryDesc *query_desc); + static EventSender *instance(); private: - EventSender(); - std::unique_ptr connector; + EventSender(); + std::unique_ptr connector; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 7329f392010..1a820404428 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -5,51 +5,43 @@ #include #include -class GrpcConnector::Impl -{ +class GrpcConnector::Impl { public: - Impl() - { - GOOGLE_PROTOBUF_VERIFY_VERSION; - this->stub = yagpcc::SetQueryInfo::NewStub(grpc::CreateChannel( - SOCKET_FILE, grpc::InsecureChannelCredentials())); + Impl() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + this->stub = yagpcc::SetQueryInfo::NewStub( + grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials())); + } + + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { + yagpcc::MetricResponse response; + grpc::ClientContext context; + auto deadline = + std::chrono::system_clock::now() + std::chrono::milliseconds(50); + context.set_deadline(deadline); + + grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); + + if (!status.ok()) { + response.set_error_text("Connection lost: " + status.error_message() + + "; " + status.error_details()); + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) - { - yagpcc::MetricResponse response; - grpc::ClientContext context; - auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(50); - context.set_deadline(deadline); - - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - - if (!status.ok()) - { - response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - } - - return response; - } + return response; + } private: - const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; - const std::string TCP_ADDRESS = "127.0.0.1:1432"; - std::unique_ptr stub; + const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; + const std::string TCP_ADDRESS = "127.0.0.1:1432"; + std::unique_ptr stub; }; -GrpcConnector::GrpcConnector() -{ - impl = new Impl(); -} +GrpcConnector::GrpcConnector() { impl = new Impl(); } -GrpcConnector::~GrpcConnector() -{ - delete impl; -} +GrpcConnector::~GrpcConnector() { delete impl; } -yagpcc::MetricResponse GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) -{ - return impl->set_metric_query(req); +yagpcc::MetricResponse +GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) { + return impl->set_metric_query(req); } \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index dc0f21706a3..810c0bd3e15 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -2,14 +2,13 @@ #include "yagpcc_set_service.pb.h" -class GrpcConnector -{ +class GrpcConnector { public: - GrpcConnector(); - ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + GrpcConnector(); + ~GrpcConnector(); + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); private: - class Impl; - Impl *impl; + class Impl; + Impl *impl; }; \ No newline at end of file diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 34c5d05719e..5c64f25ec09 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -4,116 +4,109 @@ #include #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "utils/elog.h" } namespace { -#define FILL_IO_STAT(stat_name) \ - uint64_t stat_name; \ - proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name); +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name); -void fill_io_stats(yagpcc::SystemStat *stats) -{ - std::ifstream proc_stat("/proc/self/io"); - std::string tmp; - FILL_IO_STAT(rchar); - FILL_IO_STAT(wchar); - FILL_IO_STAT(syscr); - FILL_IO_STAT(syscw); - FILL_IO_STAT(read_bytes); - FILL_IO_STAT(write_bytes); - FILL_IO_STAT(cancelled_write_bytes); +void fill_io_stats(yagpcc::SystemStat *stats) { + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(yagpcc::SystemStat *stats) -{ - static const int UTIME_ID = 13; - static const int STIME_ID = 14; - static const int STARTTIME_ID = 21; - static const int VSIZE_ID = 22; - static const int RSS_ID = 23; - static const double tps = sysconf(_SC_CLK_TCK); +void fill_cpu_stats(yagpcc::SystemStat *stats) { + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int STARTTIME_ID = 21; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); - double uptime; - { - std::ifstream proc_stat("/proc/uptime"); - proc_stat >> uptime; - } + double uptime; + { + std::ifstream proc_stat("/proc/uptime"); + proc_stat >> uptime; + } - std::ifstream proc_stat("/proc/self/stat"); - std::string trash; - double start_time = 0; - for (int i = 0; i <= RSS_ID; ++i) - { - switch (i) - { - case UTIME_ID: - double utime; - proc_stat >> utime; - stats->set_usertimeseconds(utime / tps); - break; - case STIME_ID: - double stime; - proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps); - break; - case STARTTIME_ID: - uint64_t starttime; - proc_stat >> starttime; - start_time = static_cast(starttime) / tps; - break; - case VSIZE_ID: - uint64_t vsize; - proc_stat >> vsize; - stats->set_vsize(vsize); - break; - case RSS_ID: - uint64_t rss; - proc_stat >> rss; - // NOTE: this is a double AFAIU, need to double-check - stats->set_rss(rss); - break; - default: - proc_stat >> trash; - } - stats->set_runningtimeseconds(uptime - start_time); + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + double start_time = 0; + for (int i = 0; i <= RSS_ID; ++i) { + switch (i) { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps); + break; + case STARTTIME_ID: + uint64_t starttime; + proc_stat >> starttime; + start_time = static_cast(starttime) / tps; + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; } + stats->set_runningtimeseconds(uptime - start_time); + } } -void fill_status_stats(yagpcc::SystemStat *stats) -{ - std::ifstream proc_stat("/proc/self/status"); - std::string key, measure; - while (proc_stat >> key) - { - if (key == "VmPeak:") - { - uint64_t value; - proc_stat >> value; - stats->set_vmpeakkb(value); - proc_stat >> measure; - if (measure != "kB") - elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); - } - else if (key == "VmSize:") - { - uint64_t value; - proc_stat >> value; - stats->set_vmsizekb(value); - if (measure != "kB") - elog(FATAL, "Expected memory sizes in kB, but got in %s", measure.c_str()); - } +void fill_status_stats(yagpcc::SystemStat *stats) { + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) { + if (key == "VmPeak:") { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") { + elog(FATAL, "Expected memory sizes in kB, but got in %s", + measure.c_str()); + } + } else if (key == "VmSize:") { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") { + elog(FATAL, "Expected memory sizes in kB, but got in %s", + measure.c_str()); + } } + } } } // namespace -void fill_self_stats(yagpcc::SystemStat *stats) -{ - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); +void fill_self_stats(yagpcc::SystemStat *stats) { + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 1dabb59ab3f..739cca80f01 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,8 +1,7 @@ #include "hook_wrappers.h" #include "EventSender.h" -extern "C" -{ +extern "C" { #include "postgres.h" #include "utils/metrics_utils.h" #include "utils/elog.h" @@ -22,56 +21,48 @@ static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorFinish_hook(QueryDesc *query_desc); -#define REPLACE_HOOK(hookName) \ - previous_##hookName = hookName; \ - hookName = ya_##hookName; +#define REPLACE_HOOK(hookName) \ + previous_##hookName = hookName; \ + hookName = ya_##hookName; -void hooks_init() -{ - REPLACE_HOOK(ExecutorStart_hook); - REPLACE_HOOK(ExecutorFinish_hook); - stat_statements_parser_init(); +void hooks_init() { + REPLACE_HOOK(ExecutorStart_hook); + REPLACE_HOOK(ExecutorFinish_hook); + stat_statements_parser_init(); } -void hooks_deinit() -{ - ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorFinish_hook = ExecutorFinish_hook; - stat_statements_parser_deinit(); +void hooks_deinit() { + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; + stat_statements_parser_deinit(); } -#define CREATE_HOOK_WRAPPER(hookName, ...) \ - PG_TRY(); \ - { \ - EventSender::instance()->hookName(__VA_ARGS__); \ - } \ - PG_CATCH(); \ - { \ - ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ - PG_RE_THROW(); \ - } \ - PG_END_TRY(); \ - if (previous_##hookName##_hook) \ - (*previous_##hookName##_hook)(__VA_ARGS__); \ - else \ - standard_##hookName(__VA_ARGS__); +#define CREATE_HOOK_WRAPPER(hookName, ...) \ + PG_TRY(); \ + { EventSender::instance()->hookName(__VA_ARGS__); } \ + PG_CATCH(); \ + { \ + ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ + PG_RE_THROW(); \ + } \ + PG_END_TRY(); \ + if (previous_##hookName##_hook) \ + (*previous_##hookName##_hook)(__VA_ARGS__); \ + else \ + standard_##hookName(__VA_ARGS__); -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) -{ - CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); - PG_TRY(); - { - EventSender::instance()->ExecutorStart(query_desc, eflags); - } - PG_CATCH(); - { - ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); - PG_RE_THROW(); - } - PG_END_TRY(); +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { + CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); + PG_TRY(); + { EventSender::instance()->ExecutorStart(query_desc, eflags); } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + PG_RE_THROW(); + } + PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) -{ - CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { + CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); } \ No newline at end of file From 79408d2cde183a557573020ee565f2f3e21bdc14 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 10 Apr 2023 16:01:08 +0300 Subject: [PATCH 062/167] [yagp_hooks_collector] Switch to query_info_collect_hook and fix stability Use query_info_collect_hook for finer-grained lifecycle tracking. Fix two segfaults in early init paths. Skip hooks in UTILITY mode. General robustness improvements. --- protos/yagpcc_set_service.proto | 7 +- src/EventSender.cpp | 207 ++++++++++++------ src/EventSender.h | 13 +- src/GrpcConnector.cpp | 4 +- src/GrpcConnector.h | 2 +- src/hook_wrappers.cpp | 65 +++--- .../pg_stat_statements_ya_parser.c | 21 ++ 7 files changed, 206 insertions(+), 113 deletions(-) diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 97c5691a6f5..93c2f5a01d1 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -30,9 +30,10 @@ message SetQueryReq { QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; QueryKey query_key = 3; - QueryInfo query_info = 4; - GPMetrics query_metrics = 5; - repeated MetricPlan plan_tree = 6; + SegmentKey segment_key = 4; + QueryInfo query_info = 5; + GPMetrics query_metrics = 6; + repeated MetricPlan plan_tree = 7; } message SetPlanNodeReq { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b7c3cd70b85..5ab6bbd60df 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,29 +1,30 @@ -#include "EventSender.h" #include "GrpcConnector.h" #include "ProcStats.h" -#include "protos/yagpcc_set_service.pb.h" #include extern "C" { #include "postgres.h" + #include "access/hash.h" -#include "utils/metrics_utils.h" -#include "utils/elog.h" -#include "executor/executor.h" -#include "commands/explain.h" #include "commands/dbcommands.h" +#include "commands/explain.h" #include "commands/resgroupcmds.h" +#include "executor/executor.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" -#include "cdb/cdbvars.h" #include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" -#include "pg_stat_statements_ya_parser.h" void get_spill_info(int ssid, int ccid, int32_t *file_count, int64_t *total_bytes); } +#include "EventSender.h" + namespace { std::string *get_user_name() { @@ -102,90 +103,152 @@ void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { pfree(norm_query); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - if (query_desc->sourceText) { - set_query_text(qi, query_desc); +void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, + bool with_text, bool with_plan) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + if (query_desc->sourceText && with_text) { + set_query_text(qi, query_desc); + } + if (query_desc->plannedstmt && with_plan) { + set_query_plan(qi, query_desc); + qi->set_query_id(query_desc->plannedstmt->queryId); + } + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); } - if (query_desc->plannedstmt) { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, - &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - if (query_desc->planstate->instrument) { + if (instrument) { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + bool need_spillinfo) { + if (need_spillinfo) { + int32_t n_spill_files = 0; + int64_t n_spill_bytes = 0; + get_spill_info(gp_session_id, gp_command_count, &n_spill_files, + &n_spill_bytes); + metrics->mutable_spill()->set_filecount(n_spill_files); + metrics->mutable_spill()->set_totalbytes(n_spill_bytes); + } + if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); } +yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, + yagpcc::QueryStatus status) { + yagpcc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key(), query_desc); + set_segment_key(req.mutable_segment_key(), query_desc); + return req; +} + } // namespace -void EventSender::ExecutorStart(QueryDesc *query_desc, int /* eflags*/) { +void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + switch (status) { + case METRICS_PLAN_NODE_INITIALIZE: + case METRICS_PLAN_NODE_EXECUTING: + case METRICS_PLAN_NODE_FINISHED: + // TODO + break; + case METRICS_QUERY_SUBMIT: + collect_query_submit(reinterpret_cast(arg)); + break; + case METRICS_QUERY_START: + // no-op: executor_after_start is enough + break; + case METRICS_QUERY_DONE: + collect_query_done(reinterpret_cast(arg), "done"); + break; + case METRICS_QUERY_ERROR: + collect_query_done(reinterpret_cast(arg), "error"); + break; + case METRICS_QUERY_CANCELING: + collect_query_done(reinterpret_cast(arg), "calcelling"); + break; + case METRICS_QUERY_CANCELED: + collect_query_done(reinterpret_cast(arg), "cancelled"); + break; + case METRICS_INNER_QUERY_DONE: + // TODO + break; + default: + elog(FATAL, "Unknown query status: %d", status); + } +} + +void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); + set_query_info(req.mutable_query_info(), query_desc, false, true); + send_query_info(&req, "started"); +} + +void EventSender::collect_query_submit(QueryDesc *query_desc) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; - elog(DEBUG1, "Query %s start recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - auto result = connector->set_metric_query(req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query %s start reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } else { - elog(DEBUG1, "Query %s start successful", query_desc->sourceText); - } + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + set_query_info(req.mutable_query_info(), query_desc, true, false); + send_query_info(&req, "submit"); } -void EventSender::ExecutorFinish(QueryDesc *query_desc) { - elog(DEBUG1, "Query %s finish recording", query_desc->sourceText); - yagpcc::SetQueryReq req; - req.set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_query_info(req.mutable_query_info(), query_desc); - set_gp_metrics(req.mutable_query_metrics(), query_desc); - auto result = connector->set_metric_query(req); +void EventSender::collect_query_done(QueryDesc *query_desc, + const std::string &status) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); + set_query_info(req.mutable_query_info(), query_desc, false, false); + // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to gather + // it here. It only makes sense when doing regular stat checks. + set_gp_metrics(req.mutable_query_metrics(), query_desc, + /*need_spillinfo*/ false); + send_query_info(&req, status); +} + +void EventSender::send_query_info(yagpcc::SetQueryReq *req, + const std::string &event) { + auto result = connector->set_metric_query(*req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query %s finish reporting failed with an error %s", - query_desc->sourceText, result.error_text().c_str()); - } else { - elog(DEBUG1, "Query %s finish successful", query_desc->sourceText); + elog(WARNING, "Query {%d-%d-%d} %s reporting failed with an error %s", + req->query_key().tmid(), req->query_key().ssid(), + req->query_key().ccnt(), event.c_str(), result.error_text().c_str()); } } diff --git a/src/EventSender.h b/src/EventSender.h index d69958db9b0..9c574cba9a1 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,18 +1,25 @@ #pragma once #include +#include class GrpcConnector; - struct QueryDesc; +namespace yagpcc { +class SetQueryReq; +} class EventSender { public: - void ExecutorStart(QueryDesc *query_desc, int eflags); - void ExecutorFinish(QueryDesc *query_desc); + void executor_after_start(QueryDesc *query_desc, int eflags); + void query_metrics_collect(QueryMetricsStatus status, void *arg); static EventSender *instance(); private: + void collect_query_submit(QueryDesc *query_desc); + void collect_query_done(QueryDesc *query_desc, const std::string &status); + EventSender(); + void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 1a820404428..bca1acd9ce2 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -16,8 +16,10 @@ class GrpcConnector::Impl { yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { yagpcc::MetricResponse response; grpc::ClientContext context; + // TODO: find a more secure way to send messages than relying on a fixed + // timeout auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(50); + std::chrono::system_clock::now() + std::chrono::milliseconds(200); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index 810c0bd3e15..4fca6960a4e 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -1,6 +1,6 @@ #pragma once -#include "yagpcc_set_service.pb.h" +#include "protos/yagpcc_set_service.pb.h" class GrpcConnector { public: diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 739cca80f01..be39c953970 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,6 +1,3 @@ -#include "hook_wrappers.h" -#include "EventSender.h" - extern "C" { #include "postgres.h" #include "utils/metrics_utils.h" @@ -14,55 +11,57 @@ extern "C" { } #include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "hook_wrappers.h" +#include "EventSender.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; -static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; - -static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void ya_ExecutorFinish_hook(QueryDesc *query_desc); +static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; -#define REPLACE_HOOK(hookName) \ - previous_##hookName = hookName; \ - hookName = ya_##hookName; +static void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags); +static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); void hooks_init() { - REPLACE_HOOK(ExecutorStart_hook); - REPLACE_HOOK(ExecutorFinish_hook); + previous_ExecutorStart_hook = ExecutorStart_hook; + ExecutorStart_hook = ya_ExecutorAfterStart_hook; + previous_query_info_collect_hook = query_info_collect_hook; + query_info_collect_hook = ya_query_info_collect_hook; stat_statements_parser_init(); } void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorFinish_hook = previous_ExecutorFinish_hook; + query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); } -#define CREATE_HOOK_WRAPPER(hookName, ...) \ - PG_TRY(); \ - { EventSender::instance()->hookName(__VA_ARGS__); } \ - PG_CATCH(); \ - { \ - ereport(WARNING, (errmsg("EventSender failed in %s", #hookName))); \ - PG_RE_THROW(); \ - } \ - PG_END_TRY(); \ - if (previous_##hookName##_hook) \ - (*previous_##hookName##_hook)(__VA_ARGS__); \ - else \ - standard_##hookName(__VA_ARGS__); - -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - CREATE_HOOK_WRAPPER(ExecutorStart, query_desc, eflags); +void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { + if (previous_ExecutorStart_hook) { + (*previous_ExecutorStart_hook)(query_desc, eflags); + } else { + standard_ExecutorStart(query_desc, eflags); + } PG_TRY(); - { EventSender::instance()->ExecutorStart(query_desc, eflags); } + { EventSender::instance()->executor_after_start(query_desc, eflags); } PG_CATCH(); { - ereport(WARNING, (errmsg("EventSender failed in ExecutorStart afterhook"))); + ereport(WARNING, + (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); PG_RE_THROW(); } PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - CREATE_HOOK_WRAPPER(ExecutorFinish, query_desc); +void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { + PG_TRY(); + { EventSender::instance()->query_metrics_collect(status, arg); } + PG_CATCH(); + { + ereport(WARNING, + (errmsg("EventSender failed in ya_query_info_collect_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); + if (previous_query_info_collect_hook) { + (*previous_query_info_collect_hook)(status, arg); + } } \ No newline at end of file diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index ae79e7dc40a..737e77745df 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -205,6 +205,13 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) APP_JUMB_STRING(rte->ctename); APP_JUMB(rte->ctelevelsup); break; + /* GPDB RTEs */ + case RTE_VOID: + break; + case RTE_TABLEFUNCTION: + JumbleQuery(jstate, rte->subquery); + JumbleExpr(jstate, (Node *)rte->functions); + break; default: elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); break; @@ -609,6 +616,20 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) JumbleExpr(jstate, rtfunc->funcexpr); } break; + /* GPDB nodes */ + case T_GroupingFunc: + { + GroupingFunc *grpnode = (GroupingFunc *)node; + + JumbleExpr(jstate, (Node *)grpnode->args); + } + break; + case T_Grouping: + case T_GroupId: + case T_Integer: + case T_Value: + // TODO: no idea what to do with those + break; default: /* Only a warning, since we can stumble along anyway */ elog(WARNING, "unrecognized node type: %d", From 34b5d6d44ae7c47123f164714bb6068699ceb027 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 1 May 2023 18:44:53 +0300 Subject: [PATCH 063/167] [yagp_hooks_collector] Add debian packaging and bionic GRPC compatibility --- debian/compat | 1 + debian/control | 11 +++++++++++ debian/postinst | 8 ++++++++ debian/rules | 10 ++++++++++ src/GrpcConnector.cpp | 4 ++-- 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/postinst create mode 100644 debian/rules diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000000..ec635144f60 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..600dd4d602e --- /dev/null +++ b/debian/control @@ -0,0 +1,11 @@ +Source: greenplum-6-yagpcc-hooks-collector-1 +Section: misc +Priority: optional +Maintainer: Maxim Smyatkin +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc +Standards-Version: 3.9.8 + +Package: greenplum-6-yagpcc-hooks-collector-1 +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) +Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 00000000000..27ddfc06a7d --- /dev/null +++ b/debian/postinst @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/opt/greenplum-db-6 + +chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} diff --git a/debian/rules b/debian/rules new file mode 100644 index 00000000000..6c2c7491067 --- /dev/null +++ b/debian/rules @@ -0,0 +1,10 @@ +#!/usr/bin/make -f +# You must remove unused comment lines for the released package. +export DH_VERBOSE = 1 + + +export GPHOME := /opt/greenplum-db-6 +export PATH := $(GPHOME)/bin:$(PATH) + +%: + dh $@ diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index bca1acd9ce2..5a24d576de1 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -1,8 +1,8 @@ #include "GrpcConnector.h" #include "yagpcc_set_service.grpc.pb.h" -#include -#include +#include +#include #include class GrpcConnector::Impl { From b7d9043f665104f16d2478c22792a64ee7030dbf Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 4 May 2023 14:34:42 +0300 Subject: [PATCH 064/167] [yagp_hooks_collector] Add CDB metrics, query nesting, and configuration GUCs Add missing Greenplum node types to pg_stat_statements parser. Move stats reporting to ExecutorEnd hook. Improve GRPC failure handling. Track CDB-specific metrics and initial query nesting level. Add resource group collection. Add GUCs for controlling collection. Skip nested and utility statements by default. --- debian/control | 6 +- protos/yagpcc_metrics.proto | 1 + src/Config.cpp | 38 ++++++ src/Config.h | 12 ++ src/EventSender.cpp | 112 ++++++++++++++---- src/EventSender.h | 6 + src/GrpcConnector.cpp | 66 +++++++++-- src/hook_wrappers.cpp | 93 +++++++++++++-- .../pg_stat_statements_ya_parser.c | 29 ++++- 9 files changed, 318 insertions(+), 45 deletions(-) create mode 100644 src/Config.cpp create mode 100644 src/Config.h diff --git a/debian/control b/debian/control index 600dd4d602e..c740a8590ca 100644 --- a/debian/control +++ b/debian/control @@ -1,11 +1,11 @@ -Source: greenplum-6-yagpcc-hooks-collector-1 +Source: greenplum-6-yagpcc-hooks Section: misc Priority: optional Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc, libgrpc++1, libgrpc++-dev Standards-Version: 3.9.8 -Package: greenplum-6-yagpcc-hooks-collector-1 +Package: greenplum-6-yagpcc-hooks Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index f00f329a208..26e0a496460 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -33,6 +33,7 @@ message QueryInfo { string temlate_plan_text = 7; string userName = 8; string databaseName = 9; + string rsgname = 10; } enum PlanGenerator diff --git a/src/Config.cpp b/src/Config.cpp new file mode 100644 index 00000000000..d97e5d45984 --- /dev/null +++ b/src/Config.cpp @@ -0,0 +1,38 @@ +#include "Config.h" + +extern "C" { +#include "postgres.h" +#include "utils/builtins.h" +#include "utils/guc.h" +} + +static char *guc_uds_path = nullptr; +static bool guc_enable_analyze = true; +static bool guc_enable_cdbstats = true; +static bool guc_enable_collector = true; + +void Config::init() { + DefineCustomStringVariable( + "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, + true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable_analyze", "Collect analyze metrics in yagpcc", 0LL, + &guc_enable_analyze, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, + &guc_enable_cdbstats, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); +} + +std::string Config::uds_path() { return guc_uds_path; } +bool Config::enable_analyze() { return guc_enable_analyze; } +bool Config::enable_cdbstats() { return guc_enable_cdbstats; } +bool Config::enable_collector() { return guc_enable_collector; } diff --git a/src/Config.h b/src/Config.h new file mode 100644 index 00000000000..117481f219b --- /dev/null +++ b/src/Config.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +class Config { +public: + static void init(); + static std::string uds_path(); + static bool enable_analyze(); + static bool enable_cdbstats(); + static bool enable_collector(); +}; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 5ab6bbd60df..55858ed5183 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,3 +1,4 @@ +#include "Config.h" #include "GrpcConnector.h" #include "ProcStats.h" #include @@ -13,6 +14,7 @@ extern "C" { #include "utils/elog.h" #include "utils/metrics_utils.h" +#include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" @@ -25,6 +27,10 @@ void get_spill_info(int ssid, int ccid, int32_t *file_count, #include "EventSender.h" +#define need_collect() \ + (nesting_level == 0 && gp_command_count != 0 && \ + query_desc->sourceText != nullptr && Config::enable_collector()) + namespace { std::string *get_user_name() { @@ -39,6 +45,21 @@ std::string *get_db_name() { return result; } +std::string *get_rg_name() { + auto userId = GetUserId(); + if (!OidIsValid(userId)) + return nullptr; + auto groupId = GetResGroupIdForRole(userId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + auto result = new std::string(rgname); + pfree(rgname); + return result; +} + int get_cur_slice_id(QueryDesc *desc) { if (!desc->estate) { return 0; @@ -103,9 +124,10 @@ void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { pfree(norm_query); } -void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, +void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, bool with_text, bool with_plan) { if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); if (query_desc->sourceText && with_text) { set_query_text(qi, query_desc); } @@ -115,6 +137,7 @@ void set_query_info(yagpcc::QueryInfo *qi, QueryDesc *query_desc, } qi->set_allocated_username(get_user_name()); qi->set_allocated_databasename(get_db_name()); + qi->set_allocated_rsgname(get_rg_name()); } } @@ -209,37 +232,79 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } } +void EventSender::executor_before_start(QueryDesc *query_desc, + int /* eflags*/) { + if (Gp_role == GP_ROLE_DISPATCH && need_collect() && + Config::enable_analyze()) { + instr_time starttime; + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + if (Config::enable_cdbstats()) { + query_desc->instrument_options |= INSTRUMENT_CDB; + + // TODO: there is a PR resolving some memory leak around auto-explain: + // https://github.com/greenplum-db/gpdb/pull/15164 + // Need to check if the memory leak applies here as well and fix it + Assert(query_desc->showstatctx == NULL); + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } + } +} + void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && + need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); + set_query_info(&req, query_desc, false, true); + send_query_info(&req, "started"); + } +} + +void EventSender::executor_end(QueryDesc *query_desc) { + if (!need_collect() || + (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } + if (query_desc->totaltime && Config::enable_analyze() && + Config::enable_cdbstats()) { + if (query_desc->estate->dispatcherState && + query_desc->estate->dispatcherState->primaryResults) { + cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, + DISPATCH_WAIT_NONE); + } + InstrEndLoop(query_desc->totaltime); + } auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); - set_query_info(req.mutable_query_info(), query_desc, false, true); - send_query_info(&req, "started"); + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); + set_query_info(&req, query_desc, false, false); + // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to + // gather it here. It only makes sense when doing regular stat checks. + set_gp_metrics(req.mutable_query_metrics(), query_desc, + /*need_spillinfo*/ false); + send_query_info(&req, "ended"); } void EventSender::collect_query_submit(QueryDesc *query_desc) { - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - set_query_info(req.mutable_query_info(), query_desc, true, false); - send_query_info(&req, "submit"); + if (need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + set_query_info(&req, query_desc, true, false); + send_query_info(&req, "submit"); + } } void EventSender::collect_query_done(QueryDesc *query_desc, const std::string &status) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - set_query_info(req.mutable_query_info(), query_desc, false, false); - // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to gather - // it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc, - /*need_spillinfo*/ false); - send_query_info(&req, status); + if (need_collect()) { + auto req = + create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); + set_query_info(&req, query_desc, false, false); + send_query_info(&req, status); + } } void EventSender::send_query_info(yagpcc::SetQueryReq *req, @@ -257,4 +322,7 @@ EventSender *EventSender::instance() { return &sender; } -EventSender::EventSender() { connector = std::make_unique(); } \ No newline at end of file +EventSender::EventSender() { + Config::init(); + connector = std::make_unique(); +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 9c574cba9a1..9e2ef992f81 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -11,8 +11,12 @@ class SetQueryReq; class EventSender { public: + void executor_before_start(QueryDesc *query_desc, int eflags); void executor_after_start(QueryDesc *query_desc, int eflags); + void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); + void incr_depth() { nesting_level++; } + void decr_depth() { nesting_level--; } static EventSender *instance(); private: @@ -22,4 +26,6 @@ class EventSender { EventSender(); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; + + int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 5a24d576de1..276c9ceb8a8 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -1,42 +1,86 @@ #include "GrpcConnector.h" +#include "Config.h" #include "yagpcc_set_service.grpc.pb.h" -#include +#include +#include #include +#include +#include #include +#include + +extern "C" { +#include "postgres.h" +#include "cdb/cdbvars.h" +} class GrpcConnector::Impl { public: - Impl() { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { GOOGLE_PROTOBUF_VERIFY_VERSION; - this->stub = yagpcc::SetQueryInfo::NewStub( - grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials())); + channel = + grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); + stub = yagpcc::SetQueryInfo::NewStub(channel); + connected = true; + done = false; + reconnect_thread = std::thread(&Impl::reconnect, this); + } + + ~Impl() { + done = true; + cv.notify_one(); + reconnect_thread.join(); } yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { yagpcc::MetricResponse response; + if (!connected) { + response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + response.set_error_text( + "Not tracing this query connection to agent has been lost"); + return response; + } grpc::ClientContext context; - // TODO: find a more secure way to send messages than relying on a fixed - // timeout + int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(200); + std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + connected = false; + cv.notify_one(); } return response; } private: - const std::string SOCKET_FILE = "unix:///tmp/yagpcc_agent.sock"; - const std::string TCP_ADDRESS = "127.0.0.1:1432"; + const std::string SOCKET_FILE; std::unique_ptr stub; + std::shared_ptr channel; + std::atomic_bool connected; + std::thread reconnect_thread; + std::condition_variable cv; + std::mutex mtx; + bool done; + + void reconnect() { + while (!done) { + { + std::unique_lock lock(mtx); + cv.wait(lock); + } + while (!connected && !done) { + auto deadline = + std::chrono::system_clock::now() + std::chrono::milliseconds(100); + connected = channel->WaitForConnected(deadline); + } + } + } }; GrpcConnector::GrpcConnector() { impl = new Impl(); } diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index be39c953970..edad5798e44 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,28 +1,42 @@ extern "C" { #include "postgres.h" -#include "utils/metrics_utils.h" -#include "utils/elog.h" #include "executor/executor.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" -#include "cdb/cdbvars.h" #include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" #include "tcop/utility.h" } -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" -#include "hook_wrappers.h" +#include "Config.h" #include "EventSender.h" +#include "hook_wrappers.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; +static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; +static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; +static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; -static void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + long count); +static void ya_ExecutorFinish_hook(QueryDesc *query_desc); +static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); void hooks_init() { previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = ya_ExecutorAfterStart_hook; + ExecutorStart_hook = ya_ExecutorStart_hook; + previous_ExecutorRun_hook = ExecutorRun_hook; + ExecutorRun_hook = ya_ExecutorRun_hook; + previous_ExecutorFinish_hook = ExecutorFinish_hook; + ExecutorFinish_hook = ya_ExecutorFinish_hook; + previous_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = ya_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; query_info_collect_hook = ya_query_info_collect_hook; stat_statements_parser_init(); @@ -30,11 +44,21 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); } -void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { +void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { + PG_TRY(); + { EventSender::instance()->executor_before_start(query_desc, eflags); } + PG_CATCH(); + { + ereport(WARNING, + (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); if (previous_ExecutorStart_hook) { (*previous_ExecutorStart_hook)(query_desc, eflags); } else { @@ -51,6 +75,59 @@ void ya_ExecutorAfterStart_hook(QueryDesc *query_desc, int eflags) { PG_END_TRY(); } +void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + long count) { + EventSender::instance()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorRun_hook) + previous_ExecutorRun_hook(query_desc, direction, count); + else + standard_ExecutorRun(query_desc, direction, count); + EventSender::instance()->decr_depth(); + } + PG_CATCH(); + { + EventSender::instance()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +void ya_ExecutorFinish_hook(QueryDesc *query_desc) { + EventSender::instance()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorFinish_hook) + previous_ExecutorFinish_hook(query_desc); + else + standard_ExecutorFinish(query_desc); + EventSender::instance()->decr_depth(); + } + PG_CATCH(); + { + EventSender::instance()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + +void ya_ExecutorEnd_hook(QueryDesc *query_desc) { + PG_TRY(); + { EventSender::instance()->executor_end(query_desc); } + PG_CATCH(); + { + ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); + PG_RE_THROW(); + } + PG_END_TRY(); + if (previous_ExecutorEnd_hook) { + (*previous_ExecutorEnd_hook)(query_desc); + } else { + standard_ExecutorEnd(query_desc); + } +} + void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { PG_TRY(); { EventSender::instance()->query_metrics_collect(status, arg); } diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 737e77745df..a37ac0ef0bf 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -617,6 +617,13 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) } break; /* GPDB nodes */ + case T_GroupingClause: + { + GroupingClause *grpnode = (GroupingClause *)node; + + JumbleExpr(jstate, (Node *)grpnode->groupsets); + } + break; case T_GroupingFunc: { GroupingFunc *grpnode = (GroupingFunc *)node; @@ -628,7 +635,27 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) case T_GroupId: case T_Integer: case T_Value: - // TODO: no idea what to do with those + // TODO:seems like nothing to do with it + break; + /* GPDB-only additions, nothing to do */ + case T_PartitionBy: + case T_PartitionElem: + case T_PartitionRangeItem: + case T_PartitionBoundSpec: + case T_PartitionSpec: + case T_PartitionValuesSpec: + case T_AlterPartitionId: + case T_AlterPartitionCmd: + case T_InheritPartitionCmd: + case T_CreateFileSpaceStmt: + case T_FileSpaceEntry: + case T_DropFileSpaceStmt: + case T_TableValueExpr: + case T_DenyLoginInterval: + case T_DenyLoginPoint: + case T_AlterTypeStmt: + case T_SetDistributionCmd: + case T_ExpandStmtSpec: break; default: /* Only a warning, since we can stumble along anyway */ From 7824420ecc92573ed7d56596d0923b6b819628dd Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 7 Jun 2023 14:58:57 +0300 Subject: [PATCH 065/167] [yagp_hooks_collector] Diff system stats per-query and improve error safety Capture /proc stats at query start and compute diff at end rather than reporting lifetime totals. Suppress error rethrows from the collector to avoid breaking other extensions. Add missing hooks deinitialization. Modernize ereport style. --- src/EventSender.cpp | 24 +++++++++---- src/ProcStats.cpp | 36 +++++++------------ src/hook_wrappers.cpp | 10 ++---- .../pg_stat_statements_ya_parser.c | 6 ++-- 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 55858ed5183..b1f85cf9f1e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -2,6 +2,7 @@ #include "GrpcConnector.h" #include "ProcStats.h" #include +#include extern "C" { #include "postgres.h" @@ -168,6 +169,8 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, } } +decltype(std::chrono::high_resolution_clock::now()) query_start_time; + void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, bool need_spillinfo) { if (need_spillinfo) { @@ -182,6 +185,10 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); + std::chrono::duration elapsed_seconds = + std::chrono::high_resolution_clock::now() - query_start_time; + metrics->mutable_systemstat()->set_runningtimeseconds( + elapsed_seconds.count()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -228,14 +235,17 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; default: - elog(FATAL, "Unknown query status: %d", status); + ereport(FATAL, (errmsg("Unknown query status: %d", status))); } } void EventSender::executor_before_start(QueryDesc *query_desc, int /* eflags*/) { - if (Gp_role == GP_ROLE_DISPATCH && need_collect() && - Config::enable_analyze()) { + if (!need_collect()) { + return; + } + query_start_time = std::chrono::high_resolution_clock::now(); + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { instr_time starttime; query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -311,9 +321,11 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, const std::string &event) { auto result = connector->set_metric_query(*req); if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - elog(WARNING, "Query {%d-%d-%d} %s reporting failed with an error %s", - req->query_key().tmid(), req->query_key().ssid(), - req->query_key().ccnt(), event.c_str(), result.error_text().c_str()); + ereport(WARNING, + (errmsg("Query {%d-%d-%d} %s reporting failed with an error %s", + req->query_key().tmid(), req->query_key().ssid(), + req->query_key().ccnt(), event.c_str(), + result.error_text().c_str()))); } } diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 5c64f25ec09..668173a0f7e 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -13,7 +13,7 @@ namespace { #define FILL_IO_STAT(stat_name) \ uint64_t stat_name; \ proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name); + stats->set_##stat_name(stat_name - stats->stat_name()); void fill_io_stats(yagpcc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/io"); @@ -30,36 +30,23 @@ void fill_io_stats(yagpcc::SystemStat *stats) { void fill_cpu_stats(yagpcc::SystemStat *stats) { static const int UTIME_ID = 13; static const int STIME_ID = 14; - static const int STARTTIME_ID = 21; static const int VSIZE_ID = 22; static const int RSS_ID = 23; static const double tps = sysconf(_SC_CLK_TCK); - double uptime; - { - std::ifstream proc_stat("/proc/uptime"); - proc_stat >> uptime; - } - std::ifstream proc_stat("/proc/self/stat"); std::string trash; - double start_time = 0; for (int i = 0; i <= RSS_ID; ++i) { switch (i) { case UTIME_ID: double utime; proc_stat >> utime; - stats->set_usertimeseconds(utime / tps); + stats->set_usertimeseconds(utime / tps - stats->usertimeseconds()); break; case STIME_ID: double stime; proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps); - break; - case STARTTIME_ID: - uint64_t starttime; - proc_stat >> starttime; - start_time = static_cast(starttime) / tps; + stats->set_kerneltimeseconds(stime / tps - stats->kerneltimeseconds()); break; case VSIZE_ID: uint64_t vsize; @@ -75,7 +62,6 @@ void fill_cpu_stats(yagpcc::SystemStat *stats) { default: proc_stat >> trash; } - stats->set_runningtimeseconds(uptime - start_time); } } @@ -89,16 +75,16 @@ void fill_status_stats(yagpcc::SystemStat *stats) { stats->set_vmpeakkb(value); proc_stat >> measure; if (measure != "kB") { - elog(FATAL, "Expected memory sizes in kB, but got in %s", - measure.c_str()); + ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", + measure.c_str()))); } } else if (key == "VmSize:") { uint64_t value; proc_stat >> value; stats->set_vmsizekb(value); if (measure != "kB") { - elog(FATAL, "Expected memory sizes in kB, but got in %s", - measure.c_str()); + ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", + measure.c_str()))); } } } @@ -106,7 +92,9 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } // namespace void fill_self_stats(yagpcc::SystemStat *stats) { - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); + static yagpcc::SystemStat prev_stats; + fill_io_stats(&prev_stats); + fill_cpu_stats(&prev_stats); + fill_status_stats(&prev_stats); + *stats = prev_stats; } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index edad5798e44..a904dc9bafd 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -44,6 +44,8 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorRun_hook = previous_ExecutorRun_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); @@ -56,7 +58,6 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); - PG_RE_THROW(); } PG_END_TRY(); if (previous_ExecutorStart_hook) { @@ -70,7 +71,6 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); - PG_RE_THROW(); } PG_END_TRY(); } @@ -116,10 +116,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { PG_TRY(); { EventSender::instance()->executor_end(query_desc); } PG_CATCH(); - { - ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); - PG_RE_THROW(); - } + { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } PG_END_TRY(); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); @@ -135,7 +132,6 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { { ereport(WARNING, (errmsg("EventSender failed in ya_query_info_collect_hook"))); - PG_RE_THROW(); } PG_END_TRY(); if (previous_query_info_collect_hook) { diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index a37ac0ef0bf..1c58d936093 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -213,7 +213,7 @@ JumbleRangeTable(pgssJumbleState *jstate, List *rtable) JumbleExpr(jstate, (Node *)rte->functions); break; default: - elog(ERROR, "unrecognized RTE kind: %d", (int)rte->rtekind); + ereport(ERROR, (errmsg("unrecognized RTE kind: %d", (int)rte->rtekind))); break; } } @@ -659,8 +659,8 @@ JumbleExpr(pgssJumbleState *jstate, Node *node) break; default: /* Only a warning, since we can stumble along anyway */ - elog(WARNING, "unrecognized node type: %d", - (int)nodeTag(node)); + ereport(WARNING, (errmsg("unrecognized node type: %d", + (int)nodeTag(node)))); break; } } From bb49d144c4f1e615455d17328b824c39220da573 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 13 Jun 2023 16:51:40 +0300 Subject: [PATCH 066/167] [yagp_hooks_collector] Fix EventSender and GrpcConnector in forked processes Delay initialization of static singletons and GRPC connections to actual query handling time rather than _PG_init, since both are incompatible with fork(). --- debian/control | 4 ++-- src/EventSender.cpp | 10 ++-------- src/EventSender.h | 6 ++---- src/GrpcConnector.cpp | 33 ++++++++++++++++++++++----------- src/hook_wrappers.cpp | 33 +++++++++++++++++++++++---------- 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/debian/control b/debian/control index c740a8590ca..07176e94be5 100644 --- a/debian/control +++ b/debian/control @@ -2,10 +2,10 @@ Source: greenplum-6-yagpcc-hooks Section: misc Priority: optional Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), protobuf-compiler, protobuf-compiler-grpc, libgrpc++1, libgrpc++-dev +Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) Standards-Version: 3.9.8 Package: greenplum-6-yagpcc-hooks Architecture: any -Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3) +Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/src/EventSender.cpp b/src/EventSender.cpp index b1f85cf9f1e..ec966e8686c 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -329,12 +329,6 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, } } -EventSender *EventSender::instance() { - static EventSender sender; - return &sender; -} +EventSender::EventSender() { connector = std::make_unique(); } -EventSender::EventSender() { - Config::init(); - connector = std::make_unique(); -} \ No newline at end of file +EventSender::~EventSender() { connector.release(); } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 9e2ef992f81..92e6937a690 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -17,15 +17,13 @@ class EventSender { void query_metrics_collect(QueryMetricsStatus status, void *arg); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } - static EventSender *instance(); + EventSender(); + ~EventSender(); private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - - EventSender(); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); std::unique_ptr connector; - int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 276c9ceb8a8..966bfb4a780 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -10,14 +10,17 @@ #include #include -extern "C" { +extern "C" +{ #include "postgres.h" #include "cdb/cdbvars.h" } -class GrpcConnector::Impl { +class GrpcConnector::Impl +{ public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) + { GOOGLE_PROTOBUF_VERIFY_VERSION; channel = grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); @@ -27,15 +30,18 @@ class GrpcConnector::Impl { reconnect_thread = std::thread(&Impl::reconnect, this); } - ~Impl() { + ~Impl() + { done = true; cv.notify_one(); reconnect_thread.join(); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) { + yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) + { yagpcc::MetricResponse response; - if (!connected) { + if (!connected) + { response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); response.set_error_text( "Not tracing this query connection to agent has been lost"); @@ -47,7 +53,8 @@ class GrpcConnector::Impl { std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { + if (!status.ok()) + { response.set_error_text("Connection lost: " + status.error_message() + "; " + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); @@ -68,13 +75,16 @@ class GrpcConnector::Impl { std::mutex mtx; bool done; - void reconnect() { - while (!done) { + void reconnect() + { + while (!done) + { { std::unique_lock lock(mtx); cv.wait(lock); } - while (!connected && !done) { + while (!connected && !done) + { auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(100); connected = channel->WaitForConnected(deadline); @@ -88,6 +98,7 @@ GrpcConnector::GrpcConnector() { impl = new Impl(); } GrpcConnector::~GrpcConnector() { delete impl; } yagpcc::MetricResponse -GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) { +GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) +{ return impl->set_metric_query(req); } \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index a904dc9bafd..66ba6547ce2 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -28,7 +28,17 @@ static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static EventSender *sender = nullptr; + +static inline EventSender *get_sender() { + if (!sender) { + sender = new EventSender(); + } + return sender; +} + void hooks_init() { + Config::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; @@ -49,11 +59,14 @@ void hooks_deinit() { ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); + if (sender) { + delete sender; + } } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { PG_TRY(); - { EventSender::instance()->executor_before_start(query_desc, eflags); } + { get_sender()->executor_before_start(query_desc, eflags); } PG_CATCH(); { ereport(WARNING, @@ -66,7 +79,7 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { standard_ExecutorStart(query_desc, eflags); } PG_TRY(); - { EventSender::instance()->executor_after_start(query_desc, eflags); } + { get_sender()->executor_after_start(query_desc, eflags); } PG_CATCH(); { ereport(WARNING, @@ -77,36 +90,36 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, long count) { - EventSender::instance()->incr_depth(); + get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorRun_hook) previous_ExecutorRun_hook(query_desc, direction, count); else standard_ExecutorRun(query_desc, direction, count); - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); } PG_CATCH(); { - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); PG_RE_THROW(); } PG_END_TRY(); } void ya_ExecutorFinish_hook(QueryDesc *query_desc) { - EventSender::instance()->incr_depth(); + get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorFinish_hook) previous_ExecutorFinish_hook(query_desc); else standard_ExecutorFinish(query_desc); - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); } PG_CATCH(); { - EventSender::instance()->decr_depth(); + get_sender()->decr_depth(); PG_RE_THROW(); } PG_END_TRY(); @@ -114,7 +127,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { void ya_ExecutorEnd_hook(QueryDesc *query_desc) { PG_TRY(); - { EventSender::instance()->executor_end(query_desc); } + { get_sender()->executor_end(query_desc); } PG_CATCH(); { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } PG_END_TRY(); @@ -127,7 +140,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { PG_TRY(); - { EventSender::instance()->query_metrics_collect(status, arg); } + { get_sender()->query_metrics_collect(status, arg); } PG_CATCH(); { ereport(WARNING, From 9eb39d3e2e07a65162e9e1fc718beedb9fab5378 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 16 Aug 2023 13:23:00 +0300 Subject: [PATCH 067/167] [yagp_hooks_collector] Fix memory leak in EXPLAIN ANALYZE code path --- protos/yagpcc_metrics.proto | 4 ++-- src/EventSender.cpp | 25 ++++++++----------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 26e0a496460..bc128a22f17 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -29,8 +29,8 @@ message QueryInfo { uint64 plan_id = 3; string query_text = 4; string plan_text = 5; - string temlate_query_text = 6; - string temlate_plan_text = 7; + string template_query_text = 6; + string template_plan_text = 7; string userName = 8; string databaseName = 9; string rsgname = 10; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index ec966e8686c..6d2ff4afd47 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -61,13 +61,6 @@ std::string *get_rg_name() { return result; } -int get_cur_slice_id(QueryDesc *desc) { - if (!desc->estate) { - return 0; - } - return LocallyExecutingSliceIndex(desc->estate); -} - google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -113,7 +106,7 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); set_plan_text(qi->mutable_plan_text(), query_desc); StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_temlate_plan_text() = std::string(norm_plan->data); + *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); // TODO: free stringinfo? } @@ -121,7 +114,7 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { *qi->mutable_query_text() = query_desc->sourceText; char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_temlate_query_text() = std::string(norm_query); + *qi->mutable_template_query_text() = std::string(norm_query); pfree(norm_query); } @@ -246,20 +239,18 @@ void EventSender::executor_before_start(QueryDesc *query_desc, } query_start_time = std::chrono::high_resolution_clock::now(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { - instr_time starttime; query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - // TODO: there is a PR resolving some memory leak around auto-explain: - // https://github.com/greenplum-db/gpdb/pull/15164 - // Need to check if the memory leak applies here as well and fix it - Assert(query_desc->showstatctx == NULL); - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + if (!query_desc->showstatctx) { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } } } } From 619d05a34c708beb7970ab9d68568aeb50ab405f Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 6 Sep 2023 16:10:04 +0300 Subject: [PATCH 068/167] [yagp_hooks_collector] Add motion network and workfile spill stats --- protos/yagpcc_metrics.proto | 8 ++++++++ src/EventSender.cpp | 41 ++++++++++++++++++++++++++++--------- src/EventSender.h | 2 +- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index bc128a22f17..2d20d3c46d9 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -84,6 +84,12 @@ message SystemStat { uint64 cancelled_write_bytes = 14; } +message NetworkStat { + uint32 total_bytes = 1; + uint32 tuple_bytes = 2; + uint32 chunks = 3; +} + message MetricInstrumentation { uint64 ntuples = 1; /* Total tuples produced */ uint64 nloops = 2; /* # of run cycles for this node */ @@ -103,6 +109,8 @@ message MetricInstrumentation { uint64 temp_blks_written = 16; double blk_read_time = 17; /* measured read/write time */ double blk_write_time = 18; + NetworkStat sent = 19; + NetworkStat received = 20; } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 6d2ff4afd47..2810e581313 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -4,6 +4,8 @@ #include #include +#define typeid __typeid +#define operator __operator extern "C" { #include "postgres.h" @@ -14,10 +16,12 @@ extern "C" { #include "executor/executor.h" #include "utils/elog.h" #include "utils/metrics_utils.h" +#include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/cdbinterconnect.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" @@ -25,6 +29,8 @@ extern "C" { void get_spill_info(int ssid, int ccid, int32_t *file_count, int64_t *total_bytes); } +#undef typeid +#undef operator #include "EventSender.h" @@ -160,6 +166,18 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, metrics->set_blk_write_time( INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); } + if (query_desc->estate && query_desc->estate->motionlayer_context) { + MotionLayerState *mlstate = + (MotionLayerState *)query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); + } } decltype(std::chrono::high_resolution_clock::now()) query_start_time; @@ -182,6 +200,8 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, std::chrono::high_resolution_clock::now() - query_start_time; metrics->mutable_systemstat()->set_runningtimeseconds( elapsed_seconds.count()); + metrics->mutable_spill()->set_filecount(WorkfileTotalFilesCreated()); + metrics->mutable_spill()->set_totalbytes(WorkfileTotalBytesWritten()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -238,6 +258,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } query_start_time = std::chrono::high_resolution_clock::now(); + WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -245,12 +266,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - if (!query_desc->showstatctx) { - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); - } + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -281,7 +300,6 @@ void EventSender::executor_end(QueryDesc *query_desc) { } auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); - set_query_info(&req, query_desc, false, false); // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to // gather it here. It only makes sense when doing regular stat checks. set_gp_metrics(req.mutable_query_metrics(), query_desc, @@ -303,7 +321,6 @@ void EventSender::collect_query_done(QueryDesc *query_desc, if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - set_query_info(&req, query_desc, false, false); send_query_info(&req, status); } } @@ -320,6 +337,10 @@ void EventSender::send_query_info(yagpcc::SetQueryReq *req, } } -EventSender::EventSender() { connector = std::make_unique(); } +EventSender::EventSender() { + if (Config::enable_collector()) { + connector = new GrpcConnector(); + } +} -EventSender::~EventSender() { connector.release(); } \ No newline at end of file +EventSender::~EventSender() { delete connector; } \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 92e6937a690..f53648bed36 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -24,6 +24,6 @@ class EventSender { void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); - std::unique_ptr connector; + GrpcConnector *connector; int nesting_level = 0; }; \ No newline at end of file From 51251cd7ff422e323f41898eeb41a7b6b34d46b6 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 6 Sep 2023 16:11:06 +0300 Subject: [PATCH 069/167] [yagp_hooks_collector] Clean up threading, signal handling, and logging Mute PG-destined signals in GRPC reconnection thread. Move debian config to CI. Redirect debug output to log file. Harden memory handling. Remove thread-unsafe logging and dead code. --- debian/compat | 1 - debian/control | 11 ------ debian/postinst | 8 ---- debian/rules | 10 ----- src/EventSender.cpp | 85 +++++++++++++++++++----------------------- src/EventSender.h | 1 - src/GrpcConnector.cpp | 85 ++++++++++++++++++++++++++++-------------- src/GrpcConnector.h | 3 +- src/SpillInfoWrapper.c | 21 ----------- 9 files changed, 97 insertions(+), 128 deletions(-) delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/postinst delete mode 100644 debian/rules delete mode 100644 src/SpillInfoWrapper.c diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f60..00000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index 07176e94be5..00000000000 --- a/debian/control +++ /dev/null @@ -1,11 +0,0 @@ -Source: greenplum-6-yagpcc-hooks -Section: misc -Priority: optional -Maintainer: Maxim Smyatkin -Build-Depends: make, gcc, g++, debhelper (>=9), greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) -Standards-Version: 3.9.8 - -Package: greenplum-6-yagpcc-hooks -Architecture: any -Depends: ${misc:Depends}, ${shlibs:Depends}, greenplum-db-6 (>=6.19.3), ya-grpc (=1.46-57-50820-02384e3918-yandex) -Description: Greenplum extension to send query execution metrics to yandex command center agent diff --git a/debian/postinst b/debian/postinst deleted file mode 100644 index 27ddfc06a7d..00000000000 --- a/debian/postinst +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -GPADMIN=gpadmin -GPHOME=/opt/greenplum-db-6 - -chown -R ${GPADMIN}:${GPADMIN} ${GPHOME} diff --git a/debian/rules b/debian/rules deleted file mode 100644 index 6c2c7491067..00000000000 --- a/debian/rules +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/make -f -# You must remove unused comment lines for the released package. -export DH_VERBOSE = 1 - - -export GPHOME := /opt/greenplum-db-6 -export PATH := $(GPHOME)/bin:$(PATH) - -%: - dh $@ diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 2810e581313..57fe6f13391 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,8 +1,8 @@ #include "Config.h" #include "GrpcConnector.h" #include "ProcStats.h" -#include #include +#include #define typeid __typeid #define operator __operator @@ -20,14 +20,11 @@ extern "C" { #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" -#include "cdb/cdbvars.h" #include "cdb/cdbinterconnect.h" +#include "cdb/cdbvars.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" #include "tcop/utility.h" - -void get_spill_info(int ssid, int ccid, int32_t *file_count, - int64_t *total_bytes); } #undef typeid #undef operator @@ -48,7 +45,6 @@ std::string *get_user_name() { std::string *get_db_name() { char *dbname = get_database_name(MyDatabaseId); std::string *result = dbname ? new std::string(dbname) : nullptr; - pfree(dbname); return result; } @@ -63,7 +59,6 @@ std::string *get_rg_name() { if (rgname == nullptr) return nullptr; auto result = new std::string(rgname); - pfree(rgname); return result; } @@ -114,14 +109,12 @@ void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - // TODO: free stringinfo? } void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { *qi->mutable_query_text() = query_desc->sourceText; char *norm_query = gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = std::string(norm_query); - pfree(norm_query); } void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, @@ -182,16 +175,7 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, decltype(std::chrono::high_resolution_clock::now()) query_start_time; -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, - bool need_spillinfo) { - if (need_spillinfo) { - int32_t n_spill_files = 0; - int64_t n_spill_bytes = 0; - get_spill_info(gp_session_id, gp_command_count, &n_spill_files, - &n_spill_bytes); - metrics->mutable_spill()->set_filecount(n_spill_files); - metrics->mutable_spill()->set_totalbytes(n_spill_bytes); - } +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } @@ -254,6 +238,9 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { void EventSender::executor_before_start(QueryDesc *query_desc, int /* eflags*/) { + if (!connector) { + return; + } if (!need_collect()) { return; } @@ -275,71 +262,75 @@ void EventSender::executor_before_start(QueryDesc *query_desc, } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { + if (!connector) { + return; + } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); set_query_info(&req, query_desc, false, true); - send_query_info(&req, "started"); + connector->set_metric_query(req, "started"); } } void EventSender::executor_end(QueryDesc *query_desc) { + if (!connector) { + return; + } if (!need_collect() || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } - if (query_desc->totaltime && Config::enable_analyze() && - Config::enable_cdbstats()) { - if (query_desc->estate->dispatcherState && - query_desc->estate->dispatcherState->primaryResults) { - cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, - DISPATCH_WAIT_NONE); - } - InstrEndLoop(query_desc->totaltime); - } + /* TODO: when querying via CURSOR this call freezes. Need to investigate. + To reproduce - uncomment it and run installchecks. It will freeze around join test. + Needs investigation + + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + Config::enable_cdbstats() && query_desc->estate->dispatcherState && + query_desc->estate->dispatcherState->primaryResults) { + cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, + DISPATCH_WAIT_NONE); + }*/ auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to // gather it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc, - /*need_spillinfo*/ false); - send_query_info(&req, "ended"); + set_gp_metrics(req.mutable_query_metrics(), query_desc); + connector->set_metric_query(req, "ended"); } void EventSender::collect_query_submit(QueryDesc *query_desc) { + if (!connector) { + return; + } if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); set_query_info(&req, query_desc, true, false); - send_query_info(&req, "submit"); + connector->set_metric_query(req, "submit"); } } void EventSender::collect_query_done(QueryDesc *query_desc, const std::string &status) { + if (!connector) { + return; + } if (need_collect()) { auto req = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - send_query_info(&req, status); - } -} - -void EventSender::send_query_info(yagpcc::SetQueryReq *req, - const std::string &event) { - auto result = connector->set_metric_query(*req); - if (result.error_code() == yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR) { - ereport(WARNING, - (errmsg("Query {%d-%d-%d} %s reporting failed with an error %s", - req->query_key().tmid(), req->query_key().ssid(), - req->query_key().ccnt(), event.c_str(), - result.error_text().c_str()))); + connector->set_metric_query(req, status); } } EventSender::EventSender() { if (Config::enable_collector()) { - connector = new GrpcConnector(); + try { + connector = new GrpcConnector(); + } catch (const std::exception &e) { + ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); + } } } diff --git a/src/EventSender.h b/src/EventSender.h index f53648bed36..ee0db2f0938 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,7 +23,6 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - void send_query_info(yagpcc::SetQueryReq *req, const std::string &event); GrpcConnector *connector; int nesting_level = 0; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp index 966bfb4a780..73c1944fa04 100644 --- a/src/GrpcConnector.cpp +++ b/src/GrpcConnector.cpp @@ -7,45 +7,72 @@ #include #include #include +#include +#include #include #include -extern "C" -{ +extern "C" { #include "postgres.h" #include "cdb/cdbvars.h" } -class GrpcConnector::Impl -{ +/* + * Set up the thread signal mask, we don't want to run our signal handlers + * in downloading and uploading threads. + */ +static void MaskThreadSignals() { + sigset_t sigs; + + if (pthread_equal(main_tid, pthread_self())) { + ereport(ERROR, (errmsg("thread_mask is called from main thread!"))); + return; + } + + sigemptyset(&sigs); + + /* make our thread to ignore these signals (which should allow that they be + * delivered to the main thread) */ + sigaddset(&sigs, SIGHUP); + sigaddset(&sigs, SIGINT); + sigaddset(&sigs, SIGTERM); + sigaddset(&sigs, SIGALRM); + sigaddset(&sigs, SIGUSR1); + sigaddset(&sigs, SIGUSR2); + + pthread_sigmask(SIG_BLOCK, &sigs, NULL); +} + +class GrpcConnector::Impl { public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) - { + Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { GOOGLE_PROTOBUF_VERIFY_VERSION; channel = grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); stub = yagpcc::SetQueryInfo::NewStub(channel); connected = true; + reconnected = false; done = false; reconnect_thread = std::thread(&Impl::reconnect, this); } - ~Impl() - { + ~Impl() { done = true; cv.notify_one(); reconnect_thread.join(); } - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req) - { + yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event) { yagpcc::MetricResponse response; - if (!connected) - { + if (!connected) { response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); response.set_error_text( - "Not tracing this query connection to agent has been lost"); + "Not tracing this query because grpc connection has been lost"); return response; + } else if (reconnected) { + reconnected = false; + ereport(LOG, (errmsg("GRPC connection is restored"))); } grpc::ClientContext context; int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; @@ -53,12 +80,16 @@ class GrpcConnector::Impl std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); context.set_deadline(deadline); grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) - { - response.set_error_text("Connection lost: " + status.error_message() + - "; " + status.error_details()); + if (!status.ok()) { + response.set_error_text("GRPC error: " + status.error_message() + "; " + + status.error_details()); response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str(), + response.error_text().c_str()))); connected = false; + reconnected = false; cv.notify_one(); } @@ -69,25 +100,23 @@ class GrpcConnector::Impl const std::string SOCKET_FILE; std::unique_ptr stub; std::shared_ptr channel; - std::atomic_bool connected; + std::atomic_bool connected, reconnected, done; std::thread reconnect_thread; std::condition_variable cv; std::mutex mtx; - bool done; - void reconnect() - { - while (!done) - { + void reconnect() { + MaskThreadSignals(); + while (!done) { { std::unique_lock lock(mtx); cv.wait(lock); } - while (!connected && !done) - { + while (!connected && !done) { auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(100); connected = channel->WaitForConnected(deadline); + reconnected = connected.load(); } } } @@ -98,7 +127,7 @@ GrpcConnector::GrpcConnector() { impl = new Impl(); } GrpcConnector::~GrpcConnector() { delete impl; } yagpcc::MetricResponse -GrpcConnector::set_metric_query(yagpcc::SetQueryReq req) -{ - return impl->set_metric_query(req); +GrpcConnector::set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event) { + return impl->set_metric_query(req, event); } \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h index 4fca6960a4e..6571c626dfd 100644 --- a/src/GrpcConnector.h +++ b/src/GrpcConnector.h @@ -6,7 +6,8 @@ class GrpcConnector { public: GrpcConnector(); ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(yagpcc::SetQueryReq req); + yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, + const std::string &event); private: class Impl; diff --git a/src/SpillInfoWrapper.c b/src/SpillInfoWrapper.c deleted file mode 100644 index c6ace0a693f..00000000000 --- a/src/SpillInfoWrapper.c +++ /dev/null @@ -1,21 +0,0 @@ -#include "postgres.h" -#include "utils/workfile_mgr.h" - -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes); - -void get_spill_info(int ssid, int ccid, int32_t* file_count, int64_t* total_bytes) -{ - int count = 0; - int i = 0; - workfile_set *workfiles = workfile_mgr_cache_entries_get_copy(&count); - workfile_set *wf_iter = workfiles; - for (i = 0; i < count; ++i, ++wf_iter) - { - if (wf_iter->active && wf_iter->session_id == ssid && wf_iter->command_count == ccid) - { - *file_count += wf_iter->num_files; - *total_bytes += wf_iter->total_bytes; - } - } - pfree(workfiles); -} \ No newline at end of file From 34baa67c7e6cb3c960c01ffc18bbcd58d112fb77 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 21 Sep 2023 15:16:35 +0300 Subject: [PATCH 070/167] [yagp_hooks_collector] Add ignored_users_list GUC Add a comma-separated GUC to suppress metrics collection for specified roles. Parse using SplitIdentifierString and cache in an unordered_set. --- src/Config.cpp | 43 +++++++++++++++++++++++++++++++++++++++++++ src/Config.h | 1 + src/EventSender.cpp | 5 +++-- src/EventSender.h | 2 +- 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index d97e5d45984..c5c2c15f7e9 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,7 @@ #include "Config.h" +#include +#include +#include extern "C" { #include "postgres.h" @@ -10,6 +13,8 @@ static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; +static char *guc_ignored_users = nullptr; +static std::unique_ptr> ignored_users = nullptr; void Config::init() { DefineCustomStringVariable( @@ -30,9 +35,47 @@ void Config::init() { "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomStringVariable( + "yagpcc.ignored_users_list", + "Make yagpcc ignore queries issued by given users", 0LL, + &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); } std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } + +bool Config::filter_user(const std::string *username) { + if (!ignored_users) { + ignored_users.reset(new std::unordered_set()); + if (guc_ignored_users == nullptr || guc_ignored_users[0] == '0') { + return false; + } + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return false; + } + foreach (l, elemlist) { + ignored_users->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + return !username || ignored_users->find(*username) != ignored_users->end(); +} diff --git a/src/Config.h b/src/Config.h index 117481f219b..999d0300640 100644 --- a/src/Config.h +++ b/src/Config.h @@ -9,4 +9,5 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); + static bool filter_user(const std::string *username); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 57fe6f13391..9146078fd0e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -33,7 +33,8 @@ extern "C" { #define need_collect() \ (nesting_level == 0 && gp_command_count != 0 && \ - query_desc->sourceText != nullptr && Config::enable_collector()) + query_desc->sourceText != nullptr && Config::enable_collector() && \ + !Config::filter_user(get_user_name())) namespace { @@ -325,7 +326,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } EventSender::EventSender() { - if (Config::enable_collector()) { + if (Config::enable_collector() && !Config::filter_user(get_user_name())) { try { connector = new GrpcConnector(); } catch (const std::exception &e) { diff --git a/src/EventSender.h b/src/EventSender.h index ee0db2f0938..2af8b7ffa03 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,6 +23,6 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - GrpcConnector *connector; + GrpcConnector *connector = nullptr; int nesting_level = 0; }; \ No newline at end of file From fbdb70a20ea540cd73ff956979339b4f6c4829d8 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 2 Oct 2023 12:54:32 +0300 Subject: [PATCH 071/167] [yagp_hooks_collector] Replace GRPC transport with protobuf-over-UDS Remove GRPC dependency. Serialize metrics as protobuf messages and deliver them over a Unix domain socket. Replace server-side message queue with incremental per-query message building. Add clang-format configuration. Use deprecated protobuf API for bionic compatibility. --- .clang-format | 2 + protos/yagpcc_set_service.proto | 23 ++---- src/EventSender.cpp | 115 ++++++++++++++++----------- src/EventSender.h | 10 ++- src/GrpcConnector.cpp | 133 -------------------------------- src/GrpcConnector.h | 15 ---- src/UDSConnector.cpp | 83 ++++++++++++++++++++ src/UDSConnector.h | 13 ++++ 8 files changed, 183 insertions(+), 211 deletions(-) create mode 100644 .clang-format delete mode 100644 src/GrpcConnector.cpp delete mode 100644 src/GrpcConnector.h create mode 100644 src/UDSConnector.cpp create mode 100644 src/UDSConnector.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000000..99130575c9a --- /dev/null +++ b/.clang-format @@ -0,0 +1,2 @@ +BasedOnStyle: LLVM +SortIncludes: false diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index 93c2f5a01d1..e8fc7aaa99d 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -9,23 +9,6 @@ package yagpcc; option java_outer_classname = "SegmentYAGPCCAS"; option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; -service SetQueryInfo { - rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} - - rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} -} - -message MetricResponse { - MetricResponseStatusCode error_code = 1; - string error_text = 2; -} - -enum MetricResponseStatusCode { - METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; - METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; - METRIC_RESPONSE_STATUS_CODE_ERROR = 2; -} - message SetQueryReq { QueryStatus query_status = 1; google.protobuf.Timestamp datetime = 2; @@ -34,6 +17,9 @@ message SetQueryReq { QueryInfo query_info = 5; GPMetrics query_metrics = 6; repeated MetricPlan plan_tree = 7; + google.protobuf.Timestamp submit_time = 8; + google.protobuf.Timestamp start_time = 9; + google.protobuf.Timestamp end_time = 10; } message SetPlanNodeReq { @@ -43,4 +29,7 @@ message SetPlanNodeReq { SegmentKey segment_key = 4; GPMetrics node_metrics = 5; MetricPlan plan_node = 6; + google.protobuf.Timestamp submit_time = 7; + google.protobuf.Timestamp start_time = 8; + google.protobuf.Timestamp end_time = 9; } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 9146078fd0e..834553a6187 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,6 @@ #include "Config.h" -#include "GrpcConnector.h" #include "ProcStats.h" +#include "UDSConnector.h" #include #include @@ -15,7 +15,6 @@ extern "C" { #include "commands/resgroupcmds.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/metrics_utils.h" #include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" @@ -102,33 +101,46 @@ void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { *plan_text = std::string(es.str->data, es.str->len); } -void set_query_plan(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_template_plan_text() = std::string(norm_plan->data); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + set_plan_text(qi->mutable_plan_text(), query_desc); + StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); + *qi->mutable_template_plan_text() = std::string(norm_plan->data); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + // TODO: For now assume queryid equal to planid, which is wrong. The + // reason for doing so this bug + // https://github.com/greenplum-db/gpdb/pull/15385 (ORCA loses + // pg_stat_statements` queryid during planning phase). Need to fix it + // upstream, cherry-pick and bump gp + // qi->set_query_id(query_desc->plannedstmt->queryId); + qi->set_query_id(qi->plan_id()); + } } -void set_query_text(yagpcc::QueryInfo *qi, QueryDesc *query_desc) { - *qi->mutable_query_text() = query_desc->sourceText; - char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = std::string(norm_query); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = query_desc->sourceText; + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_template_query_text() = std::string(norm_query); + } } -void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc, - bool with_text, bool with_plan) { +void clear_big_fields(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_query_text(); + } +} + +void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); - if (query_desc->sourceText && with_text) { - set_query_text(qi, query_desc); - } - if (query_desc->plannedstmt && with_plan) { - set_query_plan(qi, query_desc); - qi->set_query_id(query_desc->plannedstmt->queryId); - } qi->set_allocated_username(get_user_name()); qi->set_allocated_databasename(get_db_name()); qi->set_allocated_rsgname(get_rg_name()); @@ -245,6 +257,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!need_collect()) { return; } + if (query_msg->has_query_key()) { + connector->report_query(*query_msg, "previous query"); + query_msg->Clear(); + } query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -268,10 +284,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && need_collect()) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_START); - set_query_info(&req, query_desc, false, true); - connector->set_metric_query(req, "started"); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + *query_msg->mutable_start_time() = current_ts(); + set_query_plan(query_msg, query_desc); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } } } @@ -284,21 +302,21 @@ void EventSender::executor_end(QueryDesc *query_desc) { return; } /* TODO: when querying via CURSOR this call freezes. Need to investigate. - To reproduce - uncomment it and run installchecks. It will freeze around join test. - Needs investigation - + To reproduce - uncomment it and run installchecks. It will freeze around + join test. Needs investigation + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && Config::enable_cdbstats() && query_desc->estate->dispatcherState && query_desc->estate->dispatcherState->primaryResults) { cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, DISPATCH_WAIT_NONE); }*/ - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_END); - // NOTE: there are no cummulative spillinfo stats AFAIU, so no need to - // gather it here. It only makes sense when doing regular stat checks. - set_gp_metrics(req.mutable_query_metrics(), query_desc); - connector->set_metric_query(req, "ended"); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + if (connector->report_query(*query_msg, "ended")) { + query_msg->Clear(); + } } void EventSender::collect_query_submit(QueryDesc *query_desc) { @@ -306,10 +324,14 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { return; } if (need_collect()) { - auto req = + *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - set_query_info(&req, query_desc, true, false); - connector->set_metric_query(req, "submit"); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg, query_desc); + set_query_text(query_msg, query_desc); + if (connector->report_query(*query_msg, "submit")) { + clear_big_fields(query_msg); + } } } @@ -319,20 +341,25 @@ void EventSender::collect_query_done(QueryDesc *query_desc, return; } if (need_collect()) { - auto req = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_DONE); - connector->set_metric_query(req, status); + query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + if (connector->report_query(*query_msg, status)) { + clear_big_fields(query_msg); + } } } EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { + query_msg = new yagpcc::SetQueryReq(); try { - connector = new GrpcConnector(); + connector = new UDSConnector(); } catch (const std::exception &e) { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } } } -EventSender::~EventSender() { delete connector; } \ No newline at end of file +EventSender::~EventSender() { + delete query_msg; + delete connector; +} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 2af8b7ffa03..161bf6ce037 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,9 +1,14 @@ #pragma once #include +#include #include -class GrpcConnector; +extern "C" { +#include "utils/metrics_utils.h" +} + +class UDSConnector; struct QueryDesc; namespace yagpcc { class SetQueryReq; @@ -23,6 +28,7 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, const std::string &status); - GrpcConnector *connector = nullptr; + UDSConnector *connector = nullptr; int nesting_level = 0; + yagpcc::SetQueryReq *query_msg; }; \ No newline at end of file diff --git a/src/GrpcConnector.cpp b/src/GrpcConnector.cpp deleted file mode 100644 index 73c1944fa04..00000000000 --- a/src/GrpcConnector.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "GrpcConnector.h" -#include "Config.h" -#include "yagpcc_set_service.grpc.pb.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "postgres.h" -#include "cdb/cdbvars.h" -} - -/* - * Set up the thread signal mask, we don't want to run our signal handlers - * in downloading and uploading threads. - */ -static void MaskThreadSignals() { - sigset_t sigs; - - if (pthread_equal(main_tid, pthread_self())) { - ereport(ERROR, (errmsg("thread_mask is called from main thread!"))); - return; - } - - sigemptyset(&sigs); - - /* make our thread to ignore these signals (which should allow that they be - * delivered to the main thread) */ - sigaddset(&sigs, SIGHUP); - sigaddset(&sigs, SIGINT); - sigaddset(&sigs, SIGTERM); - sigaddset(&sigs, SIGALRM); - sigaddset(&sigs, SIGUSR1); - sigaddset(&sigs, SIGUSR2); - - pthread_sigmask(SIG_BLOCK, &sigs, NULL); -} - -class GrpcConnector::Impl { -public: - Impl() : SOCKET_FILE("unix://" + Config::uds_path()) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - channel = - grpc::CreateChannel(SOCKET_FILE, grpc::InsecureChannelCredentials()); - stub = yagpcc::SetQueryInfo::NewStub(channel); - connected = true; - reconnected = false; - done = false; - reconnect_thread = std::thread(&Impl::reconnect, this); - } - - ~Impl() { - done = true; - cv.notify_one(); - reconnect_thread.join(); - } - - yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event) { - yagpcc::MetricResponse response; - if (!connected) { - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - response.set_error_text( - "Not tracing this query because grpc connection has been lost"); - return response; - } else if (reconnected) { - reconnected = false; - ereport(LOG, (errmsg("GRPC connection is restored"))); - } - grpc::ClientContext context; - int timeout = Gp_role == GP_ROLE_DISPATCH ? 500 : 250; - auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(timeout); - context.set_deadline(deadline); - grpc::Status status = (stub->SetMetricQuery)(&context, req, &response); - if (!status.ok()) { - response.set_error_text("GRPC error: " + status.error_message() + "; " + - status.error_details()); - response.set_error_code(yagpcc::METRIC_RESPONSE_STATUS_CODE_ERROR); - ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str(), - response.error_text().c_str()))); - connected = false; - reconnected = false; - cv.notify_one(); - } - - return response; - } - -private: - const std::string SOCKET_FILE; - std::unique_ptr stub; - std::shared_ptr channel; - std::atomic_bool connected, reconnected, done; - std::thread reconnect_thread; - std::condition_variable cv; - std::mutex mtx; - - void reconnect() { - MaskThreadSignals(); - while (!done) { - { - std::unique_lock lock(mtx); - cv.wait(lock); - } - while (!connected && !done) { - auto deadline = - std::chrono::system_clock::now() + std::chrono::milliseconds(100); - connected = channel->WaitForConnected(deadline); - reconnected = connected.load(); - } - } - } -}; - -GrpcConnector::GrpcConnector() { impl = new Impl(); } - -GrpcConnector::~GrpcConnector() { delete impl; } - -yagpcc::MetricResponse -GrpcConnector::set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event) { - return impl->set_metric_query(req, event); -} \ No newline at end of file diff --git a/src/GrpcConnector.h b/src/GrpcConnector.h deleted file mode 100644 index 6571c626dfd..00000000000 --- a/src/GrpcConnector.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "protos/yagpcc_set_service.pb.h" - -class GrpcConnector { -public: - GrpcConnector(); - ~GrpcConnector(); - yagpcc::MetricResponse set_metric_query(const yagpcc::SetQueryReq &req, - const std::string &event); - -private: - class Impl; - Impl *impl; -}; \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp new file mode 100644 index 00000000000..339a5d4f374 --- /dev/null +++ b/src/UDSConnector.cpp @@ -0,0 +1,83 @@ +#include "UDSConnector.h" +#include "Config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "postgres.h" +#include "cdb/cdbvars.h" +} + +UDSConnector::UDSConnector() : uds_path("unix://" + Config::uds_path()) { + GOOGLE_PROTOBUF_VERIFY_VERSION; +} + +static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, + const std::string &event) { + ereport(LOG, + (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str(), strerror(errno)))); +} + +bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, + const std::string &event) { + sockaddr_un address; + address.sun_family = AF_UNIX; + strcpy(address.sun_path, uds_path.c_str()); + bool success = true; + auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd != -1) { + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) != -1) { + if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { + auto data_size = req.ByteSize(); + auto total_size = data_size + sizeof(uint32_t); + uint8_t *buf = (uint8_t *)palloc(total_size); + uint32_t *size_payload = (uint32_t *)buf; + *size_payload = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + int64_t sent = 0, sent_total = 0; + do { + sent = send(sockfd, buf + sent_total, total_size - sent_total, + MSG_DONTWAIT); + sent_total += sent; + } while ( + sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + if (sent < 0) { + log_tracing_failure(req, event); + success = false; + } + pfree(buf); + } else { + // log the error and go on + log_tracing_failure(req, event); + success = false; + } + } else { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %s", + strerror(errno)))); + success = false; + } + close(sockfd); + } else { + // log the error and go on + log_tracing_failure(req, event); + success = false; + } + return success; +} \ No newline at end of file diff --git a/src/UDSConnector.h b/src/UDSConnector.h new file mode 100644 index 00000000000..574653023e6 --- /dev/null +++ b/src/UDSConnector.h @@ -0,0 +1,13 @@ +#pragma once + +#include "protos/yagpcc_set_service.pb.h" +#include + +class UDSConnector { +public: + UDSConnector(); + bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); + +private: + const std::string uds_path; +}; \ No newline at end of file From 053361f20f98d8969fd91824e08360af3b55f268 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 2 Nov 2023 14:38:24 +0300 Subject: [PATCH 072/167] [yagp_hooks_collector] Fix missing query statuses after protobuf migration --- src/EventSender.cpp | 47 ++++++++++++++++++++++++++++----------------- src/EventSender.h | 2 +- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 834553a6187..45d72b93e48 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -230,16 +230,10 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // no-op: executor_after_start is enough break; case METRICS_QUERY_DONE: - collect_query_done(reinterpret_cast(arg), "done"); - break; case METRICS_QUERY_ERROR: - collect_query_done(reinterpret_cast(arg), "error"); - break; case METRICS_QUERY_CANCELING: - collect_query_done(reinterpret_cast(arg), "calcelling"); - break; case METRICS_QUERY_CANCELED: - collect_query_done(reinterpret_cast(arg), "cancelled"); + collect_query_done(reinterpret_cast(arg), status); break; case METRICS_INNER_QUERY_DONE: // TODO @@ -320,10 +314,7 @@ void EventSender::executor_end(QueryDesc *query_desc) { } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (!connector) { - return; - } - if (need_collect()) { + if (connector && need_collect()) { *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); @@ -336,13 +327,33 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { } void EventSender::collect_query_done(QueryDesc *query_desc, - const std::string &status) { - if (!connector) { - return; - } - if (need_collect()) { - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); - if (connector->report_query(*query_msg, status)) { + QueryMetricsStatus status) { + if (connector && need_collect()) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + query_msg->set_query_status(query_status); + if (connector->report_query(*query_msg, msg)) { clear_big_fields(query_msg); } } diff --git a/src/EventSender.h b/src/EventSender.h index 161bf6ce037..0e8985873b6 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -27,7 +27,7 @@ class EventSender { private: void collect_query_submit(QueryDesc *query_desc); - void collect_query_done(QueryDesc *query_desc, const std::string &status); + void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); UDSConnector *connector = nullptr; int nesting_level = 0; yagpcc::SetQueryReq *query_msg; From 8dfa3bf715f3a9942dd7142e1c74cc8de8605ee7 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 13 Nov 2023 15:38:31 +0300 Subject: [PATCH 073/167] [yagp_hooks_collector] Add stat_messages() runtime statistics view Add SQL functions stat_messages() and stat_messages_reset() exposing per-segment UDS transport counters: total_messages, send_failures, connection_failures, other_errors, max_message_size. --- sql/yagp-hooks-collector--1.0.sql | 2 - sql/yagp-hooks-collector--unpackaged--1.0.sql | 2 - sql/yagp_hooks_collector--1.0.sql | 55 +++++++++++ src/UDSConnector.cpp | 13 ++- src/UDSConnector.h | 3 - src/YagpStat.cpp | 91 +++++++++++++++++++ src/YagpStat.h | 21 +++++ src/hook_wrappers.cpp | 52 ++++++++++- src/hook_wrappers.h | 2 + src/yagp_hooks_collector.c | 13 ++- ...or.control => yagp_hooks_collector.control | 4 +- 11 files changed, 242 insertions(+), 16 deletions(-) delete mode 100644 sql/yagp-hooks-collector--1.0.sql delete mode 100644 sql/yagp-hooks-collector--unpackaged--1.0.sql create mode 100644 sql/yagp_hooks_collector--1.0.sql create mode 100644 src/YagpStat.cpp create mode 100644 src/YagpStat.h rename yagp-hooks-collector.control => yagp_hooks_collector.control (61%) diff --git a/sql/yagp-hooks-collector--1.0.sql b/sql/yagp-hooks-collector--1.0.sql deleted file mode 100644 index f9ab15fb400..00000000000 --- a/sql/yagp-hooks-collector--1.0.sql +++ /dev/null @@ -1,2 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use '''CREATE EXTENSION "yagp-hooks-collector"''' to load this file. \quit diff --git a/sql/yagp-hooks-collector--unpackaged--1.0.sql b/sql/yagp-hooks-collector--unpackaged--1.0.sql deleted file mode 100644 index 0441c97bd84..00000000000 --- a/sql/yagp-hooks-collector--unpackaged--1.0.sql +++ /dev/null @@ -1,2 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use '''CREATE EXTENSION "uuid-cb" FROM unpackaged''' to load this file. \quit diff --git a/sql/yagp_hooks_collector--1.0.sql b/sql/yagp_hooks_collector--1.0.sql new file mode 100644 index 00000000000..88bbe4e0dc7 --- /dev/null +++ b/sql/yagp_hooks_collector--1.0.sql @@ -0,0 +1,55 @@ +/* yagp_hooks_collector--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit + +CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagp_stat_messages_reset() +RETURNS void +AS +$$ + SELECT __yagp_stat_messages_reset_f_on_master(); + SELECT __yagp_stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION __yagp_stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagp_stat_messages AS + SELECT C.* + FROM __yagp_stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM __yagp_stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 339a5d4f374..b9088205250 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,5 +1,6 @@ #include "UDSConnector.h" #include "Config.h" +#include "YagpStat.h" #include #include @@ -15,9 +16,7 @@ extern "C" { #include "cdb/cdbvars.h" } -UDSConnector::UDSConnector() : uds_path("unix://" + Config::uds_path()) { - GOOGLE_PROTOBUF_VERIFY_VERSION; -} +UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { @@ -31,7 +30,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event) { sockaddr_un address; address.sun_family = AF_UNIX; - strcpy(address.sun_path, uds_path.c_str()); + strcpy(address.sun_path, Config::uds_path().c_str()); bool success = true; auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd != -1) { @@ -58,12 +57,16 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (sent < 0) { log_tracing_failure(req, event); success = false; + YagpStat::report_bad_send(total_size); + } else { + YagpStat::report_send(total_size); } pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); success = false; + YagpStat::report_bad_connection(); } } else { // That's a very important error that should never happen, so make it @@ -72,12 +75,14 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, (errmsg("Unable to create non-blocking socket connection %s", strerror(errno)))); success = false; + YagpStat::report_error(); } close(sockfd); } else { // log the error and go on log_tracing_failure(req, event); success = false; + YagpStat::report_error(); } return success; } \ No newline at end of file diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 574653023e6..42e0aa20968 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -7,7 +7,4 @@ class UDSConnector { public: UDSConnector(); bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); - -private: - const std::string uds_path; }; \ No newline at end of file diff --git a/src/YagpStat.cpp b/src/YagpStat.cpp new file mode 100644 index 00000000000..879cde85212 --- /dev/null +++ b/src/YagpStat.cpp @@ -0,0 +1,91 @@ +#include "YagpStat.h" + +#include + +extern "C" { +#include "postgres.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "storage/spin.h" +} + +namespace { +struct ProtectedData { + slock_t mutex; + YagpStat::Data data; +}; +shmem_startup_hook_type prev_shmem_startup_hook = NULL; +ProtectedData *data = nullptr; + +void yagp_shmem_startup() { + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + bool found; + data = reinterpret_cast( + ShmemInitStruct("yagp_stat_messages", sizeof(ProtectedData), &found)); + if (!found) { + SpinLockInit(&data->mutex); + data->data = YagpStat::Data(); + } + LWLockRelease(AddinShmemInitLock); +} + +class LockGuard { +public: + LockGuard(slock_t *mutex) : mutex_(mutex) { SpinLockAcquire(mutex_); } + ~LockGuard() { SpinLockRelease(mutex_); } + +private: + slock_t *mutex_; +}; +} // namespace + +void YagpStat::init() { + if (!process_shared_preload_libraries_in_progress) + return; + RequestAddinShmemSpace(sizeof(ProtectedData)); + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = yagp_shmem_startup; +} + +void YagpStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } + +void YagpStat::reset() { + LockGuard lg(&data->mutex); + data->data = YagpStat::Data(); +} + +void YagpStat::report_send(int32_t msg_size) { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +} + +void YagpStat::report_bad_connection() { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_connects++; +} + +void YagpStat::report_bad_send(int32_t msg_size) { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_sends++; + data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +} + +void YagpStat::report_error() { + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_other++; +} + +YagpStat::Data YagpStat::get_stats() { + LockGuard lg(&data->mutex); + return data->data; +} + +bool YagpStat::loaded() { return data != nullptr; } diff --git a/src/YagpStat.h b/src/YagpStat.h new file mode 100644 index 00000000000..110b1fdcbb1 --- /dev/null +++ b/src/YagpStat.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +class YagpStat { +public: + struct Data { + int64_t total, failed_sends, failed_connects, failed_other; + int32_t max_message_size; + }; + + static void init(); + static void deinit(); + static void reset(); + static void report_send(int32_t msg_size); + static void report_bad_connection(); + static void report_bad_send(int32_t msg_size); + static void report_error(); + static Data get_stats(); + static bool loaded(); +}; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 66ba6547ce2..37f80385a6b 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,16 +1,17 @@ extern "C" { #include "postgres.h" +#include "funcapi.h" #include "executor/executor.h" #include "utils/elog.h" +#include "utils/builtins.h" #include "utils/metrics_utils.h" - #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" - #include "tcop/utility.h" } #include "Config.h" +#include "YagpStat.h" #include "EventSender.h" #include "hook_wrappers.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" @@ -39,6 +40,7 @@ static inline EventSender *get_sender() { void hooks_init() { Config::init(); + YagpStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; @@ -62,6 +64,7 @@ void hooks_deinit() { if (sender) { delete sender; } + YagpStat::deinit(); } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { @@ -150,4 +153,49 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } +} + +static void check_stats_loaded() { + if (!YagpStat::loaded()) { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("yagp_hooks_collector must be loaded via " + "shared_preload_libraries"))); + } +} + +void yagp_functions_reset() { + check_stats_loaded(); + YagpStat::reset(); +} + +Datum yagp_functions_get(FunctionCallInfo fcinfo) { + const int ATTNUM = 6; + check_stats_loaded(); + auto stats = YagpStat::get_stats(); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM, false); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, + 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)3, "send_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)4, "connection_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)5, "other_errors", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)6, "max_message_size", INT4OID, + -1 /* typmod */, 0 /* attdim */); + tupdesc = BlessTupleDesc(tupdesc); + Datum values[ATTNUM]; + bool nulls[ATTNUM]; + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(GpIdentity.segindex); + values[1] = Int64GetDatum(stats.total); + values[2] = Int64GetDatum(stats.failed_sends); + values[3] = Int64GetDatum(stats.failed_connects); + values[4] = Int64GetDatum(stats.failed_other); + values[5] = Int32GetDatum(stats.max_message_size); + HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index 815fcb7cd51..c158f42cf1d 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -6,6 +6,8 @@ extern "C" { extern void hooks_init(); extern void hooks_deinit(); +extern void yagp_functions_reset(); +extern Datum yagp_functions_get(FunctionCallInfo fcinfo); #ifdef __cplusplus } diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 69475ea5079..2a9e7328e6d 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,6 +1,6 @@ #include "postgres.h" #include "cdb/cdbvars.h" -#include "fmgr.h" +#include "utils/builtins.h" #include "hook_wrappers.h" @@ -8,6 +8,8 @@ PG_MODULE_MAGIC; void _PG_init(void); void _PG_fini(void); +PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); +PG_FUNCTION_INFO_V1(yagp_stat_messages); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { @@ -20,3 +22,12 @@ void _PG_fini(void) { hooks_deinit(); } } + +Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { + yagp_functions_reset(); + PG_RETURN_VOID(); +} + +Datum yagp_stat_messages(PG_FUNCTION_ARGS) { + return yagp_functions_get(fcinfo); +} \ No newline at end of file diff --git a/yagp-hooks-collector.control b/yagp_hooks_collector.control similarity index 61% rename from yagp-hooks-collector.control rename to yagp_hooks_collector.control index 82c189a88fc..b5539dd6462 100644 --- a/yagp-hooks-collector.control +++ b/yagp_hooks_collector.control @@ -1,5 +1,5 @@ -# yagp-hooks-collector extension +# yagp_hooks_collector extension comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' default_version = '1.0' -module_pathname = '$libdir/yagp-hooks-collector' +module_pathname = '$libdir/yagp_hooks_collector' superuser = true From 66a7f854d530ee24e79f03c6d6572d90d1815ede Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 15 Nov 2023 13:37:10 +0300 Subject: [PATCH 074/167] [yagp_hooks_collector] Fix message lifecycle ordering and memory leaks Move query message cleanup to the correct lifecycle point. Finalize fields before sending DONE event. Fix protobuf message memory leaks. --- src/EventSender.cpp | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 45d72b93e48..e3be58b194e 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -39,12 +39,17 @@ namespace { std::string *get_user_name() { const char *username = GetConfigOption("session_authorization", false, false); + // username is not to be freed return username ? new std::string(username) : nullptr; } std::string *get_db_name() { char *dbname = get_database_name(MyDatabaseId); - std::string *result = dbname ? new std::string(dbname) : nullptr; + std::string *result = nullptr; + if (dbname) { + result = new std::string(dbname); + pfree(dbname); + } return result; } @@ -58,8 +63,7 @@ std::string *get_rg_name() { char *rgname = GetResGroupNameForId(groupId); if (rgname == nullptr) return nullptr; - auto result = new std::string(rgname); - return result; + return new std::string(rgname); } google::protobuf::Timestamp current_ts() { @@ -97,8 +101,12 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { } void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); *plan_text = std::string(es.str->data, es.str->len); + pfree(es.str->data); + MemoryContextSwitchTo(oldcxt); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -251,10 +259,6 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!need_collect()) { return; } - if (query_msg->has_query_key()) { - connector->report_query(*query_msg, "previous query"); - query_msg->Clear(); - } query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -263,11 +267,12 @@ void EventSender::executor_before_start(QueryDesc *query_desc, query_desc->instrument_options |= INSTRUMENT_TIMER; if (Config::enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; - - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + if (!query_desc->showstatctx) { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + cdbexplain_showExecStatsBegin(query_desc, starttime); + } } } } @@ -309,12 +314,16 @@ void EventSender::executor_end(QueryDesc *query_desc) { *query_msg->mutable_end_time() = current_ts(); set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); if (connector->report_query(*query_msg, "ended")) { - query_msg->Clear(); + clear_big_fields(query_msg); } } void EventSender::collect_query_submit(QueryDesc *query_desc) { if (connector && need_collect()) { + if (query_msg && query_msg->has_query_key()) { + connector->report_query(*query_msg, "previous query"); + query_msg->Clear(); + } *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); @@ -354,7 +363,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } query_msg->set_query_status(query_status); if (connector->report_query(*query_msg, msg)) { - clear_big_fields(query_msg); + query_msg->Clear(); } } } From 302f6419b0d8b4f41ddc3f30d93632733e9338ed Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 26 Dec 2023 16:36:26 +0300 Subject: [PATCH 075/167] [yagp_hooks_collector] Improve query_id and resource group resolution Use core query_id from Query instead of a separate hash. Resolve resource group from the current session rather than the role default. --- src/EventSender.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index e3be58b194e..21c2e2117a3 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -54,10 +54,7 @@ std::string *get_db_name() { } std::string *get_rg_name() { - auto userId = GetUserId(); - if (!OidIsValid(userId)) - return nullptr; - auto groupId = GetResGroupIdForRole(userId); + auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); if (!OidIsValid(groupId)) return nullptr; char *rgname = GetResGroupNameForId(groupId); @@ -119,13 +116,7 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); *qi->mutable_template_plan_text() = std::string(norm_plan->data); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - // TODO: For now assume queryid equal to planid, which is wrong. The - // reason for doing so this bug - // https://github.com/greenplum-db/gpdb/pull/15385 (ORCA loses - // pg_stat_statements` queryid during planning phase). Need to fix it - // upstream, cherry-pick and bump gp - // qi->set_query_id(query_desc->plannedstmt->queryId); - qi->set_query_id(qi->plan_id()); + qi->set_query_id(query_desc->plannedstmt->queryId); } } From f0a1491d2e6ae741217d48e4e4ca11c55c5c0e67 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Fri, 17 May 2024 15:55:27 +0300 Subject: [PATCH 076/167] [yagp_hooks_collector] Add nested query tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track query nesting level using a per-query key (tmid, ssid, ccnt, nesting_level, query_desc_addr). Maintain a state machine per active query to correctly sequence submit→start→end→done across nesting boundaries. --- protos/yagpcc_metrics.proto | 10 ++- protos/yagpcc_set_service.proto | 32 ++++++-- src/Config.cpp | 7 ++ src/Config.h | 1 + src/EventSender.cpp | 138 ++++++++++++++++++++++++++------ src/EventSender.h | 26 +++++- src/hook_wrappers.cpp | 2 +- 7 files changed, 178 insertions(+), 38 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 2d20d3c46d9..68492732ece 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -36,6 +36,11 @@ message QueryInfo { string rsgname = 10; } +message AdditionalQueryInfo { + int64 nested_level = 1; + string error_message = 2; +} + enum PlanGenerator { PLAN_GENERATOR_UNSPECIFIED = 0; @@ -95,7 +100,7 @@ message MetricInstrumentation { uint64 nloops = 2; /* # of run cycles for this node */ uint64 tuplecount = 3; /* Tuples emitted so far this cycle */ double firsttuple = 4; /* Time for first tuple of this cycle */ - double startup = 5; /* Total startup time (in seconds) */ + double startup = 5; /* Total startup time (in seconds) (optimiser's cost estimation) */ double total = 6; /* Total total time (in seconds) */ uint64 shared_blks_hit = 7; /* shared blocks stats*/ uint64 shared_blks_read = 8; @@ -105,12 +110,13 @@ message MetricInstrumentation { uint64 local_blks_read = 12; uint64 local_blks_dirtied = 13; uint64 local_blks_written = 14; - uint64 temp_blks_read = 15; /* temporary tables read stat */ + uint64 temp_blks_read = 15; /* temporary tables read stat */ uint64 temp_blks_written = 16; double blk_read_time = 17; /* measured read/write time */ double blk_write_time = 18; NetworkStat sent = 19; NetworkStat received = 20; + double startup_time = 21; /* real query startup time (planning + queue time) */ } message SpillInfo { diff --git a/protos/yagpcc_set_service.proto b/protos/yagpcc_set_service.proto index e8fc7aaa99d..0b9e34df49d 100644 --- a/protos/yagpcc_set_service.proto +++ b/protos/yagpcc_set_service.proto @@ -9,17 +9,35 @@ package yagpcc; option java_outer_classname = "SegmentYAGPCCAS"; option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; +service SetQueryInfo { + rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} + + rpc SetMetricQuery (SetQueryReq) returns (MetricResponse) {} +} + +message MetricResponse { + MetricResponseStatusCode error_code = 1; + string error_text = 2; +} + +enum MetricResponseStatusCode { + METRIC_RESPONSE_STATUS_CODE_UNSPECIFIED = 0; + METRIC_RESPONSE_STATUS_CODE_SUCCESS = 1; + METRIC_RESPONSE_STATUS_CODE_ERROR = 2; +} + message SetQueryReq { - QueryStatus query_status = 1; - google.protobuf.Timestamp datetime = 2; - QueryKey query_key = 3; - SegmentKey segment_key = 4; - QueryInfo query_info = 5; - GPMetrics query_metrics = 6; - repeated MetricPlan plan_tree = 7; + QueryStatus query_status = 1; + google.protobuf.Timestamp datetime = 2; + QueryKey query_key = 3; + SegmentKey segment_key = 4; + QueryInfo query_info = 5; + GPMetrics query_metrics = 6; + repeated MetricPlan plan_tree = 7; google.protobuf.Timestamp submit_time = 8; google.protobuf.Timestamp start_time = 9; google.protobuf.Timestamp end_time = 10; + AdditionalQueryInfo add_info = 11; } message SetPlanNodeReq { diff --git a/src/Config.cpp b/src/Config.cpp index c5c2c15f7e9..1bbad9a6ea3 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -13,6 +13,7 @@ static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; +static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static std::unique_ptr> ignored_users = nullptr; @@ -36,6 +37,11 @@ void Config::init() { &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomBoolVariable( + "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, + &guc_report_nested_queries, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomStringVariable( "yagpcc.ignored_users_list", "Make yagpcc ignore queries issued by given users", 0LL, @@ -47,6 +53,7 @@ std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } +bool Config::report_nested_queries() { return guc_report_nested_queries; } bool Config::filter_user(const std::string *username) { if (!ignored_users) { diff --git a/src/Config.h b/src/Config.h index 999d0300640..15f425be67c 100644 --- a/src/Config.h +++ b/src/Config.h @@ -10,4 +10,5 @@ class Config { static bool enable_cdbstats(); static bool enable_collector(); static bool filter_user(const std::string *username); + static bool report_nested_queries(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 21c2e2117a3..116805d0646 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -10,6 +10,7 @@ extern "C" { #include "postgres.h" #include "access/hash.h" +#include "access/xact.h" #include "commands/dbcommands.h" #include "commands/explain.h" #include "commands/resgroupcmds.h" @@ -30,11 +31,6 @@ extern "C" { #include "EventSender.h" -#define need_collect() \ - (nesting_level == 0 && gp_command_count != 0 && \ - query_desc->sourceText != nullptr && Config::enable_collector() && \ - !Config::filter_user(get_user_name())) - namespace { std::string *get_user_name() { @@ -146,6 +142,11 @@ void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } } +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); +} + void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; @@ -210,6 +211,19 @@ yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, return req; } +inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +inline bool need_collect(QueryDesc *query_desc, int nesting_level) { + return (Config::report_nested_queries() || + is_top_level_query(query_desc, nesting_level)) && + gp_command_count != 0 && query_desc->sourceText != nullptr && + Config::enable_collector() && !Config::filter_user(get_user_name()); +} + } // namespace void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { @@ -223,7 +237,8 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - collect_query_submit(reinterpret_cast(arg)); + // don't collect anything here. We will fake this call in ExecutorStart as + // it really makes no difference. Just complicates things break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -232,10 +247,8 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELING: case METRICS_QUERY_CANCELED: - collect_query_done(reinterpret_cast(arg), status); - break; case METRICS_INNER_QUERY_DONE: - // TODO + collect_query_done(reinterpret_cast(arg), status); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -247,9 +260,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!connector) { return; } - if (!need_collect()) { + if (!need_collect(query_desc, nesting_level)) { return; } + collect_query_submit(query_desc); query_start_time = std::chrono::high_resolution_clock::now(); WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { @@ -273,8 +287,10 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { return; } if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && - need_collect()) { - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + need_collect(query_desc, nesting_level)) { + auto *query = get_query_message(query_desc); + update_query_state(query_desc, query, QueryState::START); + auto query_msg = query->message; *query_msg->mutable_start_time() = current_ts(); set_query_plan(query_msg, query_desc); if (connector->report_query(*query_msg, "started")) { @@ -287,7 +303,7 @@ void EventSender::executor_end(QueryDesc *query_desc) { if (!connector) { return; } - if (!need_collect() || + if (!need_collect(query_desc, nesting_level) || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } @@ -301,7 +317,13 @@ void EventSender::executor_end(QueryDesc *query_desc) { cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, DISPATCH_WAIT_NONE); }*/ - query_msg->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + auto *query = get_query_message(query_desc); + if (query->state == UNKNOWN && !Config::report_nested_queries()) { + // COMMIT/ROLLBACK of a nested query. Happens in top-level + return; + } + update_query_state(query_desc, query, QueryState::END); + auto query_msg = query->message; *query_msg->mutable_end_time() = current_ts(); set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); if (connector->report_query(*query_msg, "ended")) { @@ -310,15 +332,15 @@ void EventSender::executor_end(QueryDesc *query_desc) { } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (connector && need_collect()) { - if (query_msg && query_msg->has_query_key()) { - connector->report_query(*query_msg, "previous query"); - query_msg->Clear(); - } + if (connector && need_collect(query_desc, nesting_level)) { + auto *query = get_query_message(query_desc); + query->state = QueryState::SUBMIT; + auto query_msg = query->message; *query_msg = create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg, query_desc); + set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); set_query_text(query_msg, query_desc); if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); @@ -328,11 +350,12 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && need_collect()) { + if (connector && need_collect(query_desc, nesting_level)) { yagpcc::QueryStatus query_status; std::string msg; switch (status) { case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; msg = "done"; break; @@ -352,16 +375,26 @@ void EventSender::collect_query_done(QueryDesc *query_desc, ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", status))); } - query_msg->set_query_status(query_status); - if (connector->report_query(*query_msg, msg)) { - query_msg->Clear(); + auto *query = get_query_message(query_desc); + if (query->state != UNKNOWN || Config::report_nested_queries()) { + update_query_state(query_desc, query, QueryState::DONE, + query_status == + yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query->message; + query_msg->set_query_status(query_status); + connector->report_query(*query_msg, msg); + } else { + // otherwise it`s a nested query being committed/aborted at top level + // and we should ignore it } + query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}); + pfree(query_desc->gpmon_pkt); } } EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { - query_msg = new yagpcc::SetQueryReq(); try { connector = new UDSConnector(); } catch (const std::exception &e) { @@ -371,6 +404,59 @@ EventSender::EventSender() { } EventSender::~EventSender() { - delete query_msg; delete connector; -} \ No newline at end of file + for (auto iter = query_msgs.begin(); iter != query_msgs.end(); ++iter) { + delete iter->second.message; + } +} + +// That's basically a very simplistic state machine to fix or highlight any bugs +// coming from GP +void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, + QueryState new_state, bool success) { + if (query->state == UNKNOWN) { + collect_query_submit(query_desc); + } + switch (new_state) { + case QueryState::SUBMIT: + Assert(false); + break; + case QueryState::START: + if (query->state == QueryState::SUBMIT) { + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + } else { + Assert(false); + } + break; + case QueryState::END: + Assert(query->state == QueryState::START || IsAbortInProgress()); + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + break; + case QueryState::DONE: + Assert(query->state == QueryState::END || !success); + query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + break; + default: + Assert(false); + } + query->state = new_state; +} + +EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { + if (query_desc->gpmon_pkt == nullptr || + query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}) == + query_msgs.end()) { + query_desc->gpmon_pkt = (gpmon_packet_t *)palloc0(sizeof(gpmon_packet_t)); + query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; + query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; + query_msgs.insert({{gp_command_count, nesting_level}, + QueryItem(UNKNOWN, new yagpcc::SetQueryReq())}); + } + return &query_msgs.at({query_desc->gpmon_pkt->u.qexec.key.ccnt, + query_desc->gpmon_pkt->u.qexec.key.tmid}); +} + +EventSender::QueryItem::QueryItem(EventSender::QueryState st, + yagpcc::SetQueryReq *msg) + : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 0e8985873b6..55b8daf9a91 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include extern "C" { @@ -26,9 +26,31 @@ class EventSender { ~EventSender(); private: + enum QueryState { UNKNOWN, SUBMIT, START, END, DONE }; + + struct QueryItem { + QueryState state = QueryState::UNKNOWN; + yagpcc::SetQueryReq *message = nullptr; + + QueryItem(QueryState st, yagpcc::SetQueryReq *msg); + }; + + struct pair_hash { + std::size_t operator()(const std::pair &p) const { + auto h1 = std::hash{}(p.first); + auto h2 = std::hash{}(p.second); + return h1 ^ h2; + } + }; + + void update_query_state(QueryDesc *query_desc, QueryItem *query, + QueryState new_state, bool success = true); + QueryItem *get_query_message(QueryDesc *query_desc); void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); + void cleanup_messages(); + UDSConnector *connector = nullptr; int nesting_level = 0; - yagpcc::SetQueryReq *query_msg; + std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 37f80385a6b..caf38a10f6e 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -56,9 +56,9 @@ void hooks_init() { void hooks_deinit() { ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; ExecutorRun_hook = previous_ExecutorRun_hook; ExecutorFinish_hook = previous_ExecutorFinish_hook; - ExecutorEnd_hook = previous_ExecutorEnd_hook; query_info_collect_hook = previous_query_info_collect_hook; stat_statements_parser_deinit(); if (sender) { From 68b5fad1de07a80e8726df4538cdbb93d3073e28 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 May 2024 15:25:35 +0300 Subject: [PATCH 077/167] [yagp_hooks_collector] Add configurable text field trimming Trim query text and plan text to max_text_size and max_plan_size limits. --- src/Config.cpp | 11 ++++++++++- src/Config.h | 1 + src/EventSender.cpp | 12 +++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 1bbad9a6ea3..c07a6948694 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,7 +1,8 @@ #include "Config.h" -#include +#include #include #include +#include extern "C" { #include "postgres.h" @@ -15,6 +16,7 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; +static int guc_max_text_size = 1024; // in KB static std::unique_ptr> ignored_users = nullptr; void Config::init() { @@ -47,6 +49,12 @@ void Config::init() { "Make yagpcc ignore queries issued by given users", 0LL, &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomIntVariable( + "yagpcc.max_text_size", + "Make yagpcc trim plan and query texts longer than configured size", NULL, + &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } @@ -54,6 +62,7 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } +size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { if (!ignored_users) { diff --git a/src/Config.h b/src/Config.h index 15f425be67c..f806bc0dbf5 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,4 +11,5 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); + static size_t max_text_size(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 116805d0646..4de5564533b 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -93,11 +93,15 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { return es; } +inline std::string char_to_trimmed_str(const char *str, size_t len) { + return std::string(str, std::min(len, Config::max_text_size())); +} + void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { MemoryContext oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); - *plan_text = std::string(es.str->data, es.str->len); + *plan_text = char_to_trimmed_str(es.str->data, es.str->len); pfree(es.str->data); MemoryContextSwitchTo(oldcxt); } @@ -119,9 +123,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = query_desc->sourceText; + *qi->mutable_query_text() = char_to_trimmed_str( + query_desc->sourceText, strlen(query_desc->sourceText)); char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = std::string(norm_query); + *qi->mutable_template_query_text() = + char_to_trimmed_str(norm_query, strlen(norm_query)); } } From f52364023074e902d63d6e4012c7db4985c6a3a8 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Tue, 28 May 2024 16:19:58 +0300 Subject: [PATCH 078/167] [yagp_hooks_collector] Add error message reporting for failed queries Capture elog error message at the done event for ERROR and CANCELED statuses. Properly send accumulated runtime metrics before teardown. Drop the intermediate CANCELLING event. --- src/EventSender.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 4de5564533b..8d202991986 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -153,6 +153,12 @@ void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { aqi->set_nested_level(nesting_level); } +void set_qi_error_message(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + auto error = elog_message(); + *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); +} + void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, QueryDesc *query_desc) { auto instrument = query_desc->planstate->instrument; @@ -249,9 +255,13 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_START: // no-op: executor_after_start is enough break; + case METRICS_QUERY_CANCELING: + // it appears we're unly interested in the actual CANCELED event. + // for now we will ignore CANCELING state unless otherwise requested from + // end users + break; case METRICS_QUERY_DONE: case METRICS_QUERY_ERROR: - case METRICS_QUERY_CANCELING: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: collect_query_done(reinterpret_cast(arg), status); @@ -370,6 +380,9 @@ void EventSender::collect_query_done(QueryDesc *query_desc, msg = "error"; break; case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code here + // just in case + Assert(false); query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; msg = "cancelling"; break; @@ -382,12 +395,21 @@ void EventSender::collect_query_done(QueryDesc *query_desc, status))); } auto *query = get_query_message(query_desc); + auto prev_state = query->state; if (query->state != UNKNOWN || Config::report_nested_queries()) { update_query_state(query_desc, query, QueryState::DONE, query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query->message; query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + } connector->report_query(*query_msg, msg); } else { // otherwise it`s a nested query being committed/aborted at top level @@ -435,7 +457,9 @@ void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, } break; case QueryState::END: - Assert(query->state == QueryState::START || IsAbortInProgress()); + // Example of below assert triggering: CURSOR closes before ever being + // executed Assert(query->state == QueryState::START || + // IsAbortInProgress()); query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: From ba9aa35c3d4c6e891011dd93f813cb470ed50616 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 3 Jun 2024 18:22:00 +0300 Subject: [PATCH 079/167] [yagp_hooks_collector] Change report_nested_queries to PGC_USERSET --- src/Config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.cpp b/src/Config.cpp index c07a6948694..42fa4b2fb12 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -41,7 +41,7 @@ void Config::init() { DefineCustomBoolVariable( "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, - &guc_report_nested_queries, true, PGC_SUSET, + &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomStringVariable( From 5632410c2f059a635811473f29a3bce58e025aa3 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 13 Jun 2024 10:59:46 +0300 Subject: [PATCH 080/167] [yagp_hooks_collector] Diff per-query stats between submit and end Take an initial metrics snapshot at submit so incremental stats are computed as deltas. Required for correct per-query accounting with nested statements. --- src/EventSender.cpp | 21 +++++++++++---------- src/ProcStats.cpp | 8 +++----- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 8d202991986..60f21818d00 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,7 +1,6 @@ #include "Config.h" #include "ProcStats.h" #include "UDSConnector.h" -#include #include #define typeid __typeid @@ -198,19 +197,17 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, } } -decltype(std::chrono::high_resolution_clock::now()) query_start_time; - void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); } fill_self_stats(metrics->mutable_systemstat()); - std::chrono::duration elapsed_seconds = - std::chrono::high_resolution_clock::now() - query_start_time; metrics->mutable_systemstat()->set_runningtimeseconds( - elapsed_seconds.count()); - metrics->mutable_spill()->set_filecount(WorkfileTotalFilesCreated()); - metrics->mutable_spill()->set_totalbytes(WorkfileTotalBytesWritten()); + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, @@ -280,8 +277,6 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } collect_query_submit(query_desc); - query_start_time = std::chrono::high_resolution_clock::now(); - WorkfileResetBackendStats(); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; @@ -309,9 +304,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { auto query_msg = query->message; *query_msg->mutable_start_time() = current_ts(); set_query_plan(query_msg, query_desc); + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); if (connector->report_query(*query_msg, "started")) { clear_big_fields(query_msg); } + std::swap(stats, *query_msg->mutable_query_metrics()); } } @@ -361,6 +359,9 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); } } diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 668173a0f7e..a557a20cbb0 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -92,9 +92,7 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } // namespace void fill_self_stats(yagpcc::SystemStat *stats) { - static yagpcc::SystemStat prev_stats; - fill_io_stats(&prev_stats); - fill_cpu_stats(&prev_stats); - fill_status_stats(&prev_stats); - *stats = prev_stats; + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file From bfc3a1e4575fe070b6ae664b2003e9febabb9614 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Wed, 7 Aug 2024 14:28:57 +0300 Subject: [PATCH 081/167] [yagp_hooks_collector] Fix try/catch block when calling C++ code from PG hooks --- src/hook_wrappers.cpp | 44 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index caf38a10f6e..93faaa0bf8f 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -38,6 +38,15 @@ static inline EventSender *get_sender() { return sender; } +template +R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { + try { + return (obj->*func)(args...); + } catch (const std::exception &e) { + ereport(FATAL, (errmsg("Unexpected exception in yagpcc %s", e.what()))); + } +} + void hooks_init() { Config::init(); YagpStat::init(); @@ -68,27 +77,15 @@ void hooks_deinit() { } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - PG_TRY(); - { get_sender()->executor_before_start(query_desc, eflags); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_ExecutorBeforeStart_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, + eflags); if (previous_ExecutorStart_hook) { (*previous_ExecutorStart_hook)(query_desc, eflags); } else { standard_ExecutorStart(query_desc, eflags); } - PG_TRY(); - { get_sender()->executor_after_start(query_desc, eflags); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_ExecutorAfterStart_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, + eflags); } void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -129,11 +126,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { } void ya_ExecutorEnd_hook(QueryDesc *query_desc) { - PG_TRY(); - { get_sender()->executor_end(query_desc); } - PG_CATCH(); - { ereport(WARNING, (errmsg("EventSender failed in ya_ExecutorEnd_hook"))); } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::executor_end, query_desc); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); } else { @@ -142,14 +135,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - PG_TRY(); - { get_sender()->query_metrics_collect(status, arg); } - PG_CATCH(); - { - ereport(WARNING, - (errmsg("EventSender failed in ya_query_info_collect_hook"))); - } - PG_END_TRY(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg); if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } From 6064ed4ce4518b3d98a8976a3b61eb82209aa012 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 12 Sep 2024 16:15:26 +0300 Subject: [PATCH 082/167] [yagp_hooks_collector] Improve nested query handling and add slice info Don't normalize trimmed plans. Clean up stale text fields between events. Report nested queries only from dispatcher. Add slice_id. Aggregate inherited_calls and inherited_time on segments. --- protos/yagpcc_metrics.proto | 3 + src/EventSender.cpp | 295 ++++++++++++++++++++++-------------- src/EventSender.h | 3 + 3 files changed, 191 insertions(+), 110 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 68492732ece..fc85386c6b0 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -39,6 +39,7 @@ message QueryInfo { message AdditionalQueryInfo { int64 nested_level = 1; string error_message = 2; + int64 slice_id = 3; } enum PlanGenerator @@ -117,6 +118,8 @@ message MetricInstrumentation { NetworkStat sent = 19; NetworkStat received = 20; double startup_time = 21; /* real query startup time (planning + queue time) */ + uint64 inherited_calls = 22; /* the number of executed sub-queries */ + double inherited_time = 23; /* total time spend on inherited execution */ } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 60f21818d00..7d2d5a1a2c2 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -58,6 +58,53 @@ std::string *get_rg_name() { return new std::string(rgname); } +/** + * Things get tricky with nested queries. + * a) A nested query on master is a real query optimized and executed from + * master. An example would be `select some_insert_function();`, where + * some_insert_function does something like `insert into tbl values (1)`. Master + * will create two statements. Outer select statement and inner insert statement + * with nesting level 1. + * For segments both statements are top-level statements with nesting level 0. + * b) A nested query on segment is something executed as sub-statement on + * segment. An example would be `select a from tbl where is_good_value(b);`. In + * this case master will issue one top-level statement, but segments will change + * contexts for UDF execution and execute is_good_value(b) once for each tuple + * as a nested query. Creating massive load on gpcc agent. + * + * Hence, here is a decision: + * 1) ignore all queries that are nested on segments + * 2) record (if enabled) all queries that are nested on master + * NODE: The truth is, we can't really ignore nested master queries, because + * segment sees those as top-level. + */ + +inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +inline bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return (Gp_session_role == GP_ROLE_DISPATCH && + Config::report_nested_queries()) || + is_top_level_query(query_desc, nesting_level); +} + +bool need_report_nested_query() { + return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; +} + +inline bool filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !Config::enable_collector() || Config::filter_user(get_user_name()); +} + +inline bool need_collect(QueryDesc *query_desc, int nesting_level) { + return !filter_query(query_desc) && + nesting_is_valid(query_desc, nesting_level); +} + google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -96,26 +143,24 @@ inline std::string char_to_trimmed_str(const char *str, size_t len) { return std::string(str, std::min(len, Config::max_text_size())); } -void set_plan_text(std::string *plan_text, QueryDesc *query_desc) { - MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - *plan_text = char_to_trimmed_str(es.str->data, es.str->len); - pfree(es.str->data); - MemoryContextSwitchTo(oldcxt); -} - void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - set_plan_text(qi->mutable_plan_text(), query_desc); - StringInfo norm_plan = gen_normplan(qi->plan_text().c_str()); - *qi->mutable_template_plan_text() = std::string(norm_plan->data); + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + auto es = get_explain_state(query_desc, true); + MemoryContextSwitchTo(oldcxt); + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + StringInfo norm_plan = gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = + char_to_trimmed_str(norm_plan->data, norm_plan->len); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); + pfree(es.str->data); + pfree(norm_plan->data); } } @@ -134,7 +179,9 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); + qi->clear_template_plan_text(); qi->clear_query_text(); + qi->clear_template_query_text(); } } @@ -152,6 +199,11 @@ void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { aqi->set_nested_level(nesting_level); } +void set_qi_slice_id(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); +} + void set_qi_error_message(yagpcc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); auto error = elog_message(); @@ -159,7 +211,8 @@ void set_qi_error_message(yagpcc::SetQueryReq *req) { } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, - QueryDesc *query_desc) { + QueryDesc *query_desc, int nested_calls, + double nested_time) { auto instrument = query_desc->planstate->instrument; if (instrument) { metrics->set_ntuples(instrument->ntuples); @@ -195,11 +248,15 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, mlstate->stat_tuple_bytes_recvd); metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); } -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc) { +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) { if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc); + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, + nested_calls, nested_time); } fill_self_stats(metrics->mutable_systemstat()); metrics->mutable_systemstat()->set_runningtimeseconds( @@ -220,17 +277,8 @@ yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, return req; } -inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; -} - -inline bool need_collect(QueryDesc *query_desc, int nesting_level) { - return (Config::report_nested_queries() || - is_top_level_query(query_desc, nesting_level)) && - gp_command_count != 0 && query_desc->sourceText != nullptr && - Config::enable_collector() && !Config::filter_user(get_user_name()); +double protots_to_double(const google::protobuf::Timestamp &ts) { + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } } // namespace @@ -273,6 +321,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, if (!connector) { return; } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } if (!need_collect(query_desc, nesting_level)) { return; } @@ -297,51 +349,53 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { if (!connector) { return; } - if ((Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) && - need_collect(query_desc, nesting_level)) { - auto *query = get_query_message(query_desc); - update_query_state(query_desc, query, QueryState::START); - auto query_msg = query->message; - *query_msg->mutable_start_time() = current_ts(); - set_query_plan(query_msg, query_desc); - yagpcc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { - clear_big_fields(query_msg); + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (!filter_query(query_desc)) { + auto *query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_start_time() = current_ts(); + if (!nesting_is_valid(query_desc, nesting_level)) { + return; + } + update_query_state(query_desc, query, QueryState::START); + set_query_plan(query_msg, query_desc); + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } - std::swap(stats, *query_msg->mutable_query_metrics()); } } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector) { - return; - } - if (!need_collect(query_desc, nesting_level) || + if (!connector || (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { return; } - /* TODO: when querying via CURSOR this call freezes. Need to investigate. - To reproduce - uncomment it and run installchecks. It will freeze around - join test. Needs investigation - - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && - Config::enable_cdbstats() && query_desc->estate->dispatcherState && - query_desc->estate->dispatcherState->primaryResults) { - cdbdisp_checkDispatchResult(query_desc->estate->dispatcherState, - DISPATCH_WAIT_NONE); - }*/ - auto *query = get_query_message(query_desc); - if (query->state == UNKNOWN && !Config::report_nested_queries()) { - // COMMIT/ROLLBACK of a nested query. Happens in top-level - return; - } - update_query_state(query_desc, query, QueryState::END); - auto query_msg = query->message; - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); - if (connector->report_query(*query_msg, "ended")) { - clear_big_fields(query_msg); + if (!filter_query(query_desc)) { + auto *query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_end_time() = current_ts(); + if (nesting_is_valid(query_desc, nesting_level)) { + if (query->state == UNKNOWN && + // Yet another greenplum weirdness: thats actually a nested query + // which is being committed/rollbacked. Treat it accordingly. + !need_report_nested_query()) { + return; + } + update_query_state(query_desc, query, QueryState::END); + if (is_top_level_query(query_desc, nesting_level)) { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } else { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (connector->report_query(*query_msg, "ended")) { + clear_big_fields(query_msg); + } + } } } @@ -355,66 +409,70 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg, query_desc); set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); + set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); if (connector->report_query(*query_msg, "submit")) { clear_big_fields(query_msg); } // take initial metrics snapshot so that we can safely take diff afterwards // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); } } void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && need_collect(query_desc, nesting_level)) { - yagpcc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code here - // just in case - Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, (errmsg("Unexpected query status in query_done hook: %d", - status))); - } + if (connector && !filter_query(query_desc)) { auto *query = get_query_message(query_desc); - auto prev_state = query->state; - if (query->state != UNKNOWN || Config::report_nested_queries()) { - update_query_state(query_desc, query, QueryState::DONE, - query_status == - yagpcc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query->message; - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); + if (query->state != UNKNOWN || need_report_nested_query()) { + if (nesting_is_valid(query_desc, nesting_level)) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, + (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + auto prev_state = query->state; + update_query_state(query_desc, query, QueryState::DONE, + query_status == + yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query->message; + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } + connector->report_query(*query_msg, msg); } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc); - } - connector->report_query(*query_msg, msg); - } else { - // otherwise it`s a nested query being committed/aborted at top level - // and we should ignore it + update_nested_counters(query_desc); } query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}); @@ -488,6 +546,23 @@ EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { query_desc->gpmon_pkt->u.qexec.key.tmid}); } +void EventSender::update_nested_counters(QueryDesc *query_desc) { + if (!is_top_level_query(query_desc, nesting_level)) { + auto query_msg = get_query_message(query_desc); + nested_calls++; + double end_time = protots_to_double(query_msg->message->end_time()); + double start_time = protots_to_double(query_msg->message->start_time()); + if (end_time >= start_time) { + nested_timing += end_time - start_time; + } else { + ereport(WARNING, (errmsg("YAGPCC query start_time > end_time (%f > %f)", + start_time, end_time))); + ereport(DEBUG3, + (errmsg("YAGPCC nested query text %s", query_desc->sourceText))); + } + } +} + EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 55b8daf9a91..9470cbf1f98 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -49,8 +49,11 @@ class EventSender { void collect_query_submit(QueryDesc *query_desc); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); + void update_nested_counters(QueryDesc *query_desc); UDSConnector *connector = nullptr; int nesting_level = 0; + int64_t nested_calls = 0; + double nested_timing = 0; std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file From 3ec84a2b86e71ca089403fc03e4dcb5f17c08dfa Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Thu, 7 Nov 2024 13:09:44 +0300 Subject: [PATCH 083/167] [yagp_hooks_collector] Split EventSender into submodules Factor out ProtoUtils, ProcStats, and PgUtils from EventSender. --- src/EventSender.cpp | 275 +------------------------------------------- src/PgUtils.cpp | 94 +++++++++++++++ src/PgUtils.h | 16 +++ src/ProtoUtils.cpp | 185 +++++++++++++++++++++++++++++ src/ProtoUtils.h | 16 +++ 5 files changed, 315 insertions(+), 271 deletions(-) create mode 100644 src/PgUtils.cpp create mode 100644 src/PgUtils.h create mode 100644 src/ProtoUtils.cpp create mode 100644 src/ProtoUtils.h diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 7d2d5a1a2c2..cdb21ef7aa6 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,287 +1,21 @@ #include "Config.h" -#include "ProcStats.h" #include "UDSConnector.h" -#include -#define typeid __typeid -#define operator __operator extern "C" { #include "postgres.h" #include "access/hash.h" -#include "access/xact.h" -#include "commands/dbcommands.h" -#include "commands/explain.h" -#include "commands/resgroupcmds.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/workfile_mgr.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" -#include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" - -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" -#include "tcop/utility.h" } -#undef typeid -#undef operator #include "EventSender.h" - -namespace { - -std::string *get_user_name() { - const char *username = GetConfigOption("session_authorization", false, false); - // username is not to be freed - return username ? new std::string(username) : nullptr; -} - -std::string *get_db_name() { - char *dbname = get_database_name(MyDatabaseId); - std::string *result = nullptr; - if (dbname) { - result = new std::string(dbname); - pfree(dbname); - } - return result; -} - -std::string *get_rg_name() { - auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); - if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); - if (rgname == nullptr) - return nullptr; - return new std::string(rgname); -} - -/** - * Things get tricky with nested queries. - * a) A nested query on master is a real query optimized and executed from - * master. An example would be `select some_insert_function();`, where - * some_insert_function does something like `insert into tbl values (1)`. Master - * will create two statements. Outer select statement and inner insert statement - * with nesting level 1. - * For segments both statements are top-level statements with nesting level 0. - * b) A nested query on segment is something executed as sub-statement on - * segment. An example would be `select a from tbl where is_good_value(b);`. In - * this case master will issue one top-level statement, but segments will change - * contexts for UDF execution and execute is_good_value(b) once for each tuple - * as a nested query. Creating massive load on gpcc agent. - * - * Hence, here is a decision: - * 1) ignore all queries that are nested on segments - * 2) record (if enabled) all queries that are nested on master - * NODE: The truth is, we can't really ignore nested master queries, because - * segment sees those as top-level. - */ - -inline bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; -} - -inline bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return (Gp_session_role == GP_ROLE_DISPATCH && - Config::report_nested_queries()) || - is_top_level_query(query_desc, nesting_level); -} - -bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; -} - -inline bool filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !Config::enable_collector() || Config::filter_user(get_user_name()); -} - -inline bool need_collect(QueryDesc *query_desc, int nesting_level) { - return !filter_query(query_desc) && - nesting_is_valid(query_desc, nesting_level); -} - -google::protobuf::Timestamp current_ts() { - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; -} - -void set_query_key(yagpcc::QueryKey *key, QueryDesc *query_desc) { - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gpmon_gettmid(&tmid); - key->set_tmid(tmid); -} - -void set_segment_key(yagpcc::SegmentKey *key, QueryDesc *query_desc) { - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); -} - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; -} - -inline std::string char_to_trimmed_str(const char *str, size_t len) { - return std::string(str, std::min(len, Config::max_text_size())); -} - -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { - auto qi = req->mutable_query_info(); - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); - MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); - StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = - char_to_trimmed_str(norm_plan->data, norm_plan->len); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - qi->set_query_id(query_desc->plannedstmt->queryId); - pfree(es.str->data); - pfree(norm_plan->data); - } -} - -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { - auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = char_to_trimmed_str( - query_desc->sourceText, strlen(query_desc->sourceText)); - char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = - char_to_trimmed_str(norm_query, strlen(norm_query)); - } -} - -void clear_big_fields(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->clear_plan_text(); - qi->clear_template_plan_text(); - qi->clear_query_text(); - qi->clear_template_query_text(); - } -} - -void set_query_info(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); - qi->set_allocated_rsgname(get_rg_name()); - } -} - -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { - auto aqi = req->mutable_add_info(); - aqi->set_nested_level(nesting_level); -} - -void set_qi_slice_id(yagpcc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - aqi->set_slice_id(currentSliceId); -} - -void set_qi_error_message(yagpcc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - auto error = elog_message(); - *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); -} - -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, - QueryDesc *query_desc, int nested_calls, - double nested_time) { - auto instrument = query_desc->planstate->instrument; - if (instrument) { - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time( - INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); - } - if (query_desc->estate && query_desc->estate->motionlayer_context) { - MotionLayerState *mlstate = - (MotionLayerState *)query_desc->estate->motionlayer_context; - metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); - metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); - metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); - metrics->mutable_received()->set_total_bytes( - mlstate->stat_total_bytes_recvd); - metrics->mutable_received()->set_tuple_bytes( - mlstate->stat_tuple_bytes_recvd); - metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); - } - metrics->set_inherited_calls(nested_calls); - metrics->set_inherited_time(nested_time); -} - -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time) { - if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, - nested_calls, nested_time); - } - fill_self_stats(metrics->mutable_systemstat()); - metrics->mutable_systemstat()->set_runningtimeseconds( - time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); - metrics->mutable_spill()->set_filecount( - WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); - metrics->mutable_spill()->set_totalbytes( - WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); -} - -yagpcc::SetQueryReq create_query_req(QueryDesc *query_desc, - yagpcc::QueryStatus status) { - yagpcc::SetQueryReq req; - req.set_query_status(status); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key(), query_desc); - set_segment_key(req.mutable_segment_key(), query_desc); - return req; -} - -double protots_to_double(const google::protobuf::Timestamp &ts) { - return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; -} - -} // namespace +#include "PgUtils.h" +#include "ProtoUtils.h" void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { @@ -404,10 +138,9 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { auto *query = get_query_message(query_desc); query->state = QueryState::SUBMIT; auto query_msg = query->message; - *query_msg = - create_query_req(query_desc, yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg, query_desc); + set_query_info(query_msg); set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp new file mode 100644 index 00000000000..528426e6c64 --- /dev/null +++ b/src/PgUtils.cpp @@ -0,0 +1,94 @@ +#include "PgUtils.h" +#include "Config.h" + +extern "C" { +#include "utils/guc.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" +#include "cdb/cdbvars.h" +} + +std::string *get_user_name() { + const char *username = GetConfigOption("session_authorization", false, false); + // username is not to be freed + return username ? new std::string(username) : nullptr; +} + +std::string *get_db_name() { + char *dbname = get_database_name(MyDatabaseId); + std::string *result = nullptr; + if (dbname) { + result = new std::string(dbname); + pfree(dbname); + } + return result; +} + +std::string *get_rg_name() { + auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); + if (!OidIsValid(groupId)) + return nullptr; + char *rgname = GetResGroupNameForId(groupId); + if (rgname == nullptr) + return nullptr; + return new std::string(rgname); +} + +/** + * Things get tricky with nested queries. + * a) A nested query on master is a real query optimized and executed from + * master. An example would be `select some_insert_function();`, where + * some_insert_function does something like `insert into tbl values (1)`. Master + * will create two statements. Outer select statement and inner insert statement + * with nesting level 1. + * For segments both statements are top-level statements with nesting level 0. + * b) A nested query on segment is something executed as sub-statement on + * segment. An example would be `select a from tbl where is_good_value(b);`. In + * this case master will issue one top-level statement, but segments will change + * contexts for UDF execution and execute is_good_value(b) once for each tuple + * as a nested query. Creating massive load on gpcc agent. + * + * Hence, here is a decision: + * 1) ignore all queries that are nested on segments + * 2) record (if enabled) all queries that are nested on master + * NODE: The truth is, we can't really ignore nested master queries, because + * segment sees those as top-level. + */ + +bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { + return (query_desc->gpmon_pkt && + query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || + nesting_level == 0; +} + +bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return (Gp_session_role == GP_ROLE_DISPATCH && + Config::report_nested_queries()) || + is_top_level_query(query_desc, nesting_level); +} + +bool need_report_nested_query() { + return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; +} + +bool filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !Config::enable_collector() || Config::filter_user(get_user_name()); +} + +bool need_collect(QueryDesc *query_desc, int nesting_level) { + return !filter_query(query_desc) && + nesting_is_valid(query_desc, nesting_level); +} + +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; +} diff --git a/src/PgUtils.h b/src/PgUtils.h new file mode 100644 index 00000000000..85b1eb833cd --- /dev/null +++ b/src/PgUtils.h @@ -0,0 +1,16 @@ +extern "C" { +#include "postgres.h" +#include "commands/explain.h" +} + +#include + +std::string *get_user_name(); +std::string *get_db_name(); +std::string *get_rg_name(); +bool is_top_level_query(QueryDesc *query_desc, int nesting_level); +bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); +bool need_report_nested_query(); +bool filter_query(QueryDesc *query_desc); +bool need_collect(QueryDesc *query_desc, int nesting_level); +ExplainState get_explain_state(QueryDesc *query_desc, bool costs); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp new file mode 100644 index 00000000000..e1be25b8b1e --- /dev/null +++ b/src/ProtoUtils.cpp @@ -0,0 +1,185 @@ +#include "ProtoUtils.h" +#include "PgUtils.h" +#include "ProcStats.h" +#include "Config.h" + +#define typeid __typeid +#define operator __operator +extern "C" { +#include "postgres.h" +#include "access/hash.h" +#include "cdb/cdbinterconnect.h" +#include "cdb/cdbvars.h" +#include "gpmon/gpmon.h" +#include "utils/workfile_mgr.h" + +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +} +#undef typeid +#undef operator + +#include +#include + +google::protobuf::Timestamp current_ts() { + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; +} + +void set_query_key(yagpcc::QueryKey *key) { + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gpmon_gettmid(&tmid); + key->set_tmid(tmid); +} + +void set_segment_key(yagpcc::SegmentKey *key) { + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); +} + +inline std::string char_to_trimmed_str(const char *str, size_t len) { + return std::string(str, std::min(len, Config::max_text_size())); +} + +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + auto es = get_explain_state(query_desc, true); + MemoryContextSwitchTo(oldcxt); + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + StringInfo norm_plan = gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = + char_to_trimmed_str(norm_plan->data, norm_plan->len); + qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + qi->set_query_id(query_desc->plannedstmt->queryId); + pfree(es.str->data); + pfree(norm_plan->data); + } +} + +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { + if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = char_to_trimmed_str( + query_desc->sourceText, strlen(query_desc->sourceText)); + char *norm_query = gen_normquery(query_desc->sourceText); + *qi->mutable_template_query_text() = + char_to_trimmed_str(norm_query, strlen(norm_query)); + } +} + +void clear_big_fields(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_template_plan_text(); + qi->clear_query_text(); + qi->clear_template_query_text(); + } +} + +void set_query_info(yagpcc::SetQueryReq *req) { + if (Gp_session_role == GP_ROLE_DISPATCH) { + auto qi = req->mutable_query_info(); + qi->set_allocated_username(get_user_name()); + qi->set_allocated_databasename(get_db_name()); + qi->set_allocated_rsgname(get_rg_name()); + } +} + +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); +} + +void set_qi_slice_id(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); +} + +void set_qi_error_message(yagpcc::SetQueryReq *req) { + auto aqi = req->mutable_add_info(); + auto error = elog_message(); + *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); +} + +void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, + QueryDesc *query_desc, int nested_calls, + double nested_time) { + auto instrument = query_desc->planstate->instrument; + if (instrument) { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } + if (query_desc->estate && query_desc->estate->motionlayer_context) { + MotionLayerState *mlstate = + (MotionLayerState *)query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); + } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); +} + +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) { + if (query_desc->planstate && query_desc->planstate->instrument) { + set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, + nested_calls, nested_time); + } + fill_self_stats(metrics->mutable_systemstat()); + metrics->mutable_systemstat()->set_runningtimeseconds( + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); +} + +yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { + yagpcc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key()); + set_segment_key(req.mutable_segment_key()); + return req; +} + +double protots_to_double(const google::protobuf::Timestamp &ts) { + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +} \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h new file mode 100644 index 00000000000..38aa75611b2 --- /dev/null +++ b/src/ProtoUtils.h @@ -0,0 +1,16 @@ +#include "protos/yagpcc_set_service.pb.h" + +struct QueryDesc; + +google::protobuf::Timestamp current_ts(); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void clear_big_fields(yagpcc::SetQueryReq *req); +void set_query_info(yagpcc::SetQueryReq *req); +void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); +void set_qi_slice_id(yagpcc::SetQueryReq *req); +void set_qi_error_message(yagpcc::SetQueryReq *req); +void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time); +yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); +double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file From 3e386491c342ee2aa576d705d563ee02c6ba98be Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Mon, 7 Apr 2025 14:15:39 +0300 Subject: [PATCH 084/167] [yagp_hooks_collector] Ignore EXPLAIN VERBOSE errors for unsupported node types --- src/PgUtils.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 528426e6c64..5982ff77c1c 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -88,7 +88,24 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { es.verbose = true; es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); + PG_TRY(); + { ExplainPrintPlan(&es, query_desc); } + PG_CATCH(); + { + // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE + // implementation. We don't want any queries to fail due to those bugs, so + // we report the bug here for future investigatin and continue collecting + // metrics w/o reporting any plans + resetStringInfo(es.str); + appendStringInfo( + es.str, + "Unable to restore query plan due to PostgreSQL internal error. " + "See logs for more information"); + ereport(INFO, + (errmsg("YAGPCC failed to reconstruct explain text for query: %s", + query_desc->sourceText))); + } + PG_END_TRY(); ExplainEndOutput(&es); return es; } From 112ed241a286241b50d5ee1a1ec9e7d0c0943366 Mon Sep 17 00:00:00 2001 From: Maxim Smyatkin Date: Fri, 18 Apr 2025 14:58:52 +0300 Subject: [PATCH 085/167] [yagp_hooks_collector] Add per-slice interconnect statistics Hook into ic_teardown to collect UDP-IFC packet-level counters. Compile-time gated behind IC_TEARDOWN_HOOK. --- protos/yagpcc_metrics.proto | 56 +++++++++++++++++++++++++++++++++++++ src/EventSender.cpp | 53 ++++++++++++++++++++++++++++++++++- src/EventSender.h | 10 +++++++ src/ProtoUtils.cpp | 35 +++++++++++++++++++++++ src/ProtoUtils.h | 3 ++ src/hook_wrappers.cpp | 24 ++++++++++++++++ 6 files changed, 180 insertions(+), 1 deletion(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index fc85386c6b0..086f3e63379 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -42,6 +42,11 @@ message AdditionalQueryInfo { int64 slice_id = 3; } +message AdditionalQueryStat { + string error_message = 1; + repeated int64 slices = 2; +} + enum PlanGenerator { PLAN_GENERATOR_UNSPECIFIED = 0; @@ -96,6 +101,56 @@ message NetworkStat { uint32 chunks = 3; } +message InterconnectStat { + // Receive queue size sum when main thread is trying to get a packet + uint64 total_recv_queue_size = 1; + // Counting times when computing total_recv_queue_size + uint64 recv_queue_size_counting_time = 2; + + // The capacity sum when packets are tried to be sent + uint64 total_capacity = 3; + // Counting times used to compute total_capacity + uint64 capacity_counting_time = 4; + + // Total buffers available when sending packets + uint64 total_buffers = 5; + // Counting times when compute total_buffers + uint64 buffer_counting_time = 6; + + // The number of active connections + uint64 active_connections_num = 7; + + // The number of packet retransmits + int64 retransmits = 8; + + // The number of cached future packets + int64 startup_cached_pkt_num = 9; + + // The number of mismatched packets received + int64 mismatch_num = 10; + + // The number of crc errors + int64 crc_errors = 11; + + // The number of packets sent by sender + int64 snd_pkt_num = 12; + + // The number of packets received by receiver + int64 recv_pkt_num = 13; + + // Disordered packet number + int64 disordered_pkt_num = 14; + + // Duplicate packet number + int64 duplicated_pkt_num = 15; + + // The number of Acks received + int64 recv_ack_num = 16; + + // The number of status query messages sent + int64 status_query_msg_num = 17; +} + message MetricInstrumentation { uint64 ntuples = 1; /* Total tuples produced */ uint64 nloops = 2; /* # of run cycles for this node */ @@ -120,6 +175,7 @@ message MetricInstrumentation { double startup_time = 21; /* real query startup time (planning + queue time) */ uint64 inherited_calls = 22; /* the number of executed sub-queries */ double inherited_time = 23; /* total time spend on inherited execution */ + InterconnectStat interconnect = 24; } message SpillInfo { diff --git a/src/EventSender.cpp b/src/EventSender.cpp index cdb21ef7aa6..2ba34d1e4cc 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,7 @@ #include "Config.h" #include "UDSConnector.h" +#define typeid __typeid extern "C" { #include "postgres.h" @@ -11,7 +12,9 @@ extern "C" { #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" } +#undef typeid #include "EventSender.h" #include "PgUtils.h" @@ -35,7 +38,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // no-op: executor_after_start is enough break; case METRICS_QUERY_CANCELING: - // it appears we're unly interested in the actual CANCELED event. + // it appears we're only interested in the actual CANCELED event. // for now we will ignore CANCELING state unless otherwise requested from // end users break; @@ -150,6 +153,12 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { // take initial metrics snapshot so that we can safely take diff afterwards // in END or DONE events. set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); +#ifdef IC_TEARDOWN_HOOK + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif } } @@ -203,6 +212,12 @@ void EventSender::collect_query_done(QueryDesc *query_desc, set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, nested_timing); } +#ifdef IC_TEARDOWN_HOOK + ic_metrics_collect(); + set_ic_stats( + query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif connector->report_query(*query_msg, msg); } update_nested_counters(query_desc); @@ -213,6 +228,39 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } } +void EventSender::ic_metrics_collect() { +#ifdef IC_TEARDOWN_HOOK + if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { + return; + } + if (!connector || gp_command_count == 0 || !Config::enable_collector() || + Config::filter_user(get_user_name())) { + return; + } + // we also would like to know nesting level here and filter queries BUT we + // don't have this kind of information from this callback. Will have to + // collect stats anyways and throw it away later, if necessary + auto metrics = UDPIFCGetICStats(); + ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; + ic_statistics.recvQueueSizeCountingTime += metrics.recvQueueSizeCountingTime; + ic_statistics.totalCapacity += metrics.totalCapacity; + ic_statistics.capacityCountingTime += metrics.capacityCountingTime; + ic_statistics.totalBuffers += metrics.totalBuffers; + ic_statistics.bufferCountingTime += metrics.bufferCountingTime; + ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; + ic_statistics.retransmits += metrics.retransmits; + ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; + ic_statistics.mismatchNum += metrics.mismatchNum; + ic_statistics.crcErrors += metrics.crcErrors; + ic_statistics.sndPktNum += metrics.sndPktNum; + ic_statistics.recvPktNum += metrics.recvPktNum; + ic_statistics.disorderedPktNum += metrics.disorderedPktNum; + ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; + ic_statistics.recvAckNum += metrics.recvAckNum; + ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; +#endif +} + EventSender::EventSender() { if (Config::enable_collector() && !Config::filter_user(get_user_name())) { try { @@ -221,6 +269,9 @@ EventSender::EventSender() { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } } +#ifdef IC_TEARDOWN_HOOK + memset(&ic_statistics, 0, sizeof(ICStatistics)); +#endif } EventSender::~EventSender() { diff --git a/src/EventSender.h b/src/EventSender.h index 9470cbf1f98..99f7b24753d 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -4,9 +4,15 @@ #include #include +#define typeid __typeid extern "C" { #include "utils/metrics_utils.h" +#include "cdb/ml_ipc.h" +#ifdef IC_TEARDOWN_HOOK +#include "cdb/ic_udpifc.h" +#endif } +#undef typeid class UDSConnector; struct QueryDesc; @@ -20,6 +26,7 @@ class EventSender { void executor_after_start(QueryDesc *query_desc, int eflags); void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); + void ic_metrics_collect(); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } EventSender(); @@ -55,5 +62,8 @@ class EventSender { int nesting_level = 0; int64_t nested_calls = 0; double nested_timing = 0; +#ifdef IC_TEARDOWN_HOOK + ICStatistics ic_statistics; +#endif std::unordered_map, QueryItem, pair_hash> query_msgs; }; \ No newline at end of file diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index e1be25b8b1e..c37cefb72d6 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -10,6 +10,10 @@ extern "C" { #include "access/hash.h" #include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" +#ifdef IC_TEARDOWN_HOOK +#include "cdb/ic_udpifc.h" +#endif #include "gpmon/gpmon.h" #include "utils/workfile_mgr.h" @@ -171,6 +175,37 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } +#define UPDATE_IC_STATS(proto_name, stat_name) \ + metrics->mutable_interconnect()->set_##proto_name( \ + ic_statistics->stat_name - \ + metrics->mutable_interconnect()->proto_name()); \ + Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ + metrics->mutable_interconnect()->proto_name() <= \ + ic_statistics->stat_name) + +void set_ic_stats(yagpcc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics) { +#ifdef IC_TEARDOWN_HOOK + UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); + UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); + UPDATE_IC_STATS(total_capacity, totalCapacity); + UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); + UPDATE_IC_STATS(total_buffers, totalBuffers); + UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); + UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); + UPDATE_IC_STATS(retransmits, retransmits); + UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); + UPDATE_IC_STATS(mismatch_num, mismatchNum); + UPDATE_IC_STATS(crc_errors, crcErrors); + UPDATE_IC_STATS(snd_pkt_num, sndPktNum); + UPDATE_IC_STATS(recv_pkt_num, recvPktNum); + UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); + UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); + UPDATE_IC_STATS(recv_ack_num, recvAckNum); + UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); +#endif +} + yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { yagpcc::SetQueryReq req; req.set_query_status(status); diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 38aa75611b2..4e4ed5e76a3 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,6 +1,7 @@ #include "protos/yagpcc_set_service.pb.h" struct QueryDesc; +struct ICStatistics; google::protobuf::Timestamp current_ts(); void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); @@ -12,5 +13,7 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req); void set_qi_error_message(yagpcc::SetQueryReq *req); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); +void set_ic_stats(yagpcc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 93faaa0bf8f..f1d403b82f1 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,3 +1,4 @@ +#define typeid __typeid extern "C" { #include "postgres.h" #include "funcapi.h" @@ -7,8 +8,10 @@ extern "C" { #include "utils/metrics_utils.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" +#include "cdb/ml_ipc.h" #include "tcop/utility.h" } +#undef typeid #include "Config.h" #include "YagpStat.h" @@ -21,6 +24,9 @@ static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; +#ifdef IC_TEARDOWN_HOOK +static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; +#endif static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -28,6 +34,8 @@ static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static void ya_ic_teardown_hook(ChunkTransportState *transportStates, + bool hasErrors); static EventSender *sender = nullptr; @@ -60,6 +68,10 @@ void hooks_init() { ExecutorEnd_hook = ya_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; query_info_collect_hook = ya_query_info_collect_hook; +#ifdef IC_TEARDOWN_HOOK + previous_ic_teardown_hook = ic_teardown_hook; + ic_teardown_hook = ya_ic_teardown_hook; +#endif stat_statements_parser_init(); } @@ -69,6 +81,9 @@ void hooks_deinit() { ExecutorRun_hook = previous_ExecutorRun_hook; ExecutorFinish_hook = previous_ExecutorFinish_hook; query_info_collect_hook = previous_query_info_collect_hook; +#ifdef IC_TEARDOWN_HOOK + ic_teardown_hook = previous_ic_teardown_hook; +#endif stat_statements_parser_deinit(); if (sender) { delete sender; @@ -141,6 +156,15 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } } +void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { + cpp_call(get_sender(), &EventSender::ic_metrics_collect); +#ifdef IC_TEARDOWN_HOOK + if (previous_ic_teardown_hook) { + (*previous_ic_teardown_hook)(transportStates, hasErrors); + } +#endif +} + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), From 37d9a37dee1903b068da81b906b074a1546b0192 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 9 Jun 2025 16:59:13 +0300 Subject: [PATCH 086/167] [yagp_hooks_collector] Fix user filtering propagation timing --- src/Config.cpp | 51 ++++++++++++++------------------------------- src/Config.h | 1 + src/EventSender.cpp | 40 ++++++++++++++++++++++++++++++++++- src/EventSender.h | 1 + 4 files changed, 57 insertions(+), 36 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 42fa4b2fb12..19aa37d1b9d 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -6,7 +6,6 @@ extern "C" { #include "postgres.h" -#include "utils/builtins.h" #include "utils/guc.h" } @@ -17,7 +16,12 @@ static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1024; // in KB -static std::unique_ptr> ignored_users = nullptr; +std::unique_ptr> ignored_users_set = nullptr; +bool ignored_users_guc_dirty = false; + +static void assign_ignored_users_hook(const char *, void *) { + ignored_users_guc_dirty = true; +} void Config::init() { DefineCustomStringVariable( @@ -44,11 +48,12 @@ void Config::init() { &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - DefineCustomStringVariable( - "yagpcc.ignored_users_list", - "Make yagpcc ignore queries issued by given users", 0LL, - &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + DefineCustomStringVariable("yagpcc.ignored_users_list", + "Make yagpcc ignore queries issued by given users", + 0LL, &guc_ignored_users, + "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, + assign_ignored_users_hook, 0LL); DefineCustomIntVariable( "yagpcc.max_text_size", @@ -62,36 +67,12 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } +const char *Config::ignored_users() { return guc_ignored_users; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { - if (!ignored_users) { - ignored_users.reset(new std::unordered_set()); - if (guc_ignored_users == nullptr || guc_ignored_users[0] == '0') { - return false; - } - /* Need a modifiable copy of string */ - char *rawstring = pstrdup(guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { - /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return false; - } - foreach (l, elemlist) { - ignored_users->insert((char *)lfirst(l)); - } - pfree(rawstring); - list_free(elemlist); + if (!username || !ignored_users_set) { + return true; } - return !username || ignored_users->find(*username) != ignored_users->end(); + return ignored_users_set->find(*username) != ignored_users_set->end(); } diff --git a/src/Config.h b/src/Config.h index f806bc0dbf5..9dd33c68321 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,5 +11,6 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); + static const char *ignored_users(); static size_t max_text_size(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 2ba34d1e4cc..fed9b69911f 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,6 +8,7 @@ extern "C" { #include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" +#include "utils/builtins.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" @@ -20,6 +21,9 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" +extern std::unique_ptr> ignored_users_set; +extern bool ignored_users_guc_dirty; + void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -62,6 +66,10 @@ void EventSender::executor_before_start(QueryDesc *query_desc, nested_timing = 0; nested_calls = 0; } + if (ignored_users_guc_dirty) { + update_ignored_users(Config::ignored_users()); + ignored_users_guc_dirty = false; + } if (!need_collect(query_desc, nesting_level)) { return; } @@ -262,7 +270,7 @@ void EventSender::ic_metrics_collect() { } EventSender::EventSender() { - if (Config::enable_collector() && !Config::filter_user(get_user_name())) { + if (Config::enable_collector()) { try { connector = new UDSConnector(); } catch (const std::exception &e) { @@ -347,6 +355,36 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } +void EventSender::update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = + std::make_unique>(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + ignored_users_set = std::move(new_ignored_users_set); +} + EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 99f7b24753d..6919defbbb3 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -57,6 +57,7 @@ class EventSender { void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); + void update_ignored_users(const char *new_guc_ignored_users); UDSConnector *connector = nullptr; int nesting_level = 0; From fd27ed77eb226cb0d6d6a6b78a0c6d663efa7624 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 16 Jun 2025 13:07:59 +0300 Subject: [PATCH 087/167] [yagp_hooks_collector] Miscellaneous fixes and refactoring Fix UB in strcpy. General code refactoring. --- src/Config.cpp | 44 +++++++++++++++++++++++++++++++++++++++++--- src/Config.h | 2 +- src/EventSender.cpp | 39 +-------------------------------------- src/EventSender.h | 1 - src/UDSConnector.cpp | 8 +++++++- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 19aa37d1b9d..5e0749f171d 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -6,6 +6,7 @@ extern "C" { #include "postgres.h" +#include "utils/builtins.h" #include "utils/guc.h" } @@ -16,8 +17,39 @@ static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1024; // in KB -std::unique_ptr> ignored_users_set = nullptr; -bool ignored_users_guc_dirty = false; +static std::unique_ptr> ignored_users_set = + nullptr; +static bool ignored_users_guc_dirty = false; + +static void update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = + std::make_unique>(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + /* syntax error in list */ + pfree(rawstring); + list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + pfree(rawstring); + list_free(elemlist); + } + ignored_users_set = std::move(new_ignored_users_set); +} static void assign_ignored_users_hook(const char *, void *) { ignored_users_guc_dirty = true; @@ -67,7 +99,6 @@ bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } -const char *Config::ignored_users() { return guc_ignored_users; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } bool Config::filter_user(const std::string *username) { @@ -76,3 +107,10 @@ bool Config::filter_user(const std::string *username) { } return ignored_users_set->find(*username) != ignored_users_set->end(); } + +void Config::sync() { + if (ignored_users_guc_dirty) { + update_ignored_users(guc_ignored_users); + ignored_users_guc_dirty = false; + } +} \ No newline at end of file diff --git a/src/Config.h b/src/Config.h index 9dd33c68321..3caa0c78339 100644 --- a/src/Config.h +++ b/src/Config.h @@ -11,6 +11,6 @@ class Config { static bool enable_collector(); static bool filter_user(const std::string *username); static bool report_nested_queries(); - static const char *ignored_users(); static size_t max_text_size(); + static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fed9b69911f..fc0f7e1aa07 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,7 +8,6 @@ extern "C" { #include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" -#include "utils/builtins.h" #include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" @@ -21,9 +20,6 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" -extern std::unique_ptr> ignored_users_set; -extern bool ignored_users_guc_dirty; - void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -66,10 +62,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, nested_timing = 0; nested_calls = 0; } - if (ignored_users_guc_dirty) { - update_ignored_users(Config::ignored_users()); - ignored_users_guc_dirty = false; - } + Config::sync(); if (!need_collect(query_desc, nesting_level)) { return; } @@ -355,36 +348,6 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } -void EventSender::update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = - std::make_unique>(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { - /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - pfree(rawstring); - list_free(elemlist); - } - ignored_users_set = std::move(new_ignored_users_set); -} - EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) : state(st), message(msg) {} \ No newline at end of file diff --git a/src/EventSender.h b/src/EventSender.h index 6919defbbb3..99f7b24753d 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -57,7 +57,6 @@ class EventSender { void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); - void update_ignored_users(const char *new_guc_ignored_users); UDSConnector *connector = nullptr; int nesting_level = 0; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b9088205250..8a5f754f3b4 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -30,7 +30,13 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event) { sockaddr_un address; address.sun_family = AF_UNIX; - strcpy(address.sun_path, Config::uds_path().c_str()); + std::string uds_path = Config::uds_path(); + if (uds_path.size() >= sizeof(address.sun_path)) { + ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); + YagpStat::report_error(); + return false; + } + strcpy(address.sun_path, uds_path.c_str()); bool success = true; auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd != -1) { From 608d7ba5f11497d1f28533998d3e73f80f757c94 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:41:03 +0300 Subject: [PATCH 088/167] [yagp_hooks_collector] Add conditional EXPLAIN ANALYZE collection When enable_analyze is true and execution time exceeds min_analyze_time, generate EXPLAIN (ANALYZE, BUFFERS, TIMING, VERBOSE) output and include it in the done event. --- protos/yagpcc_metrics.proto | 1 + src/Config.cpp | 22 +++++++++++++-- src/Config.h | 2 ++ src/EventSender.cpp | 51 ++++++++++++++++++++++++++++++++--- src/EventSender.h | 1 + src/PgUtils.cpp | 37 ++++++++++++++++++++++++++ src/PgUtils.h | 1 + src/ProtoUtils.cpp | 53 ++++++++++++++++++++++++++++++------- src/ProtoUtils.h | 4 ++- src/hook_wrappers.cpp | 24 +++++++++++++++++ 10 files changed, 180 insertions(+), 16 deletions(-) diff --git a/protos/yagpcc_metrics.proto b/protos/yagpcc_metrics.proto index 086f3e63379..91ac0c4941a 100644 --- a/protos/yagpcc_metrics.proto +++ b/protos/yagpcc_metrics.proto @@ -34,6 +34,7 @@ message QueryInfo { string userName = 8; string databaseName = 9; string rsgname = 10; + string analyze_text = 11; } message AdditionalQueryInfo { diff --git a/src/Config.cpp b/src/Config.cpp index 5e0749f171d..ac274a1e218 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,7 +16,10 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1024; // in KB +static int guc_max_text_size = 1024; // in KB +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = -1; // uninitialized state + static std::unique_ptr> ignored_users_set = nullptr; static bool ignored_users_guc_dirty = false; @@ -89,9 +92,22 @@ void Config::init() { DefineCustomIntVariable( "yagpcc.max_text_size", - "Make yagpcc trim plan and query texts longer than configured size", NULL, + "Make yagpcc trim query texts longer than configured size", NULL, &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "yagpcc.max_plan_size", + "Make yagpcc trim plan longer than configured size", NULL, + &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "yagpcc.min_analyze_time", + "Sets the minimum execution time above which plans will be logged.", + "Zero prints all plans. -1 turns this feature off.", + &guc_min_analyze_time, -1, -1, INT_MAX, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } @@ -100,6 +116,8 @@ bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } bool Config::report_nested_queries() { return guc_report_nested_queries; } size_t Config::max_text_size() { return guc_max_text_size * 1024; } +size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } +int Config::min_analyze_time() { return guc_min_analyze_time; }; bool Config::filter_user(const std::string *username) { if (!username || !ignored_users_set) { diff --git a/src/Config.h b/src/Config.h index 3caa0c78339..dd081c41dd6 100644 --- a/src/Config.h +++ b/src/Config.h @@ -12,5 +12,7 @@ class Config { static bool filter_user(const std::string *username); static bool report_nested_queries(); static size_t max_text_size(); + static size_t max_plan_size(); + static int min_analyze_time(); static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fc0f7e1aa07..19787fe0db0 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -20,6 +20,10 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" +#define need_collect_analyze() \ + (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ + Config::enable_analyze()) + void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; @@ -53,8 +57,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } } -void EventSender::executor_before_start(QueryDesc *query_desc, - int /* eflags*/) { +void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { if (!connector) { return; } @@ -67,7 +70,8 @@ void EventSender::executor_before_start(QueryDesc *query_desc, return; } collect_query_submit(query_desc); - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze()) { + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; @@ -97,6 +101,17 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } update_query_state(query_desc, query, QueryState::START); set_query_plan(query_msg, query_desc); + if (need_collect_analyze()) { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) { + MemoryContext oldcxt; + oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + query_desc->totaltime = InstrAlloc(1, INSTRUMENT_ALL); + MemoryContextSwitchTo(oldcxt); + } + } yagpcc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); if (connector->report_query(*query_msg, "started")) { @@ -262,6 +277,34 @@ void EventSender::ic_metrics_collect() { #endif } +void EventSender::analyze_stats_collect(QueryDesc *query_desc) { + if (!connector || Gp_role != GP_ROLE_DISPATCH) { + return; + } + if (!need_collect(query_desc, nesting_level)) { + return; + } + auto query = get_query_message(query_desc); + auto query_msg = query->message; + *query_msg->mutable_end_time() = current_ts(); + // Yet another greenplum weirdness: thats actually a nested query + // which is being committed/rollbacked. Treat it accordingly. + if (query->state == UNKNOWN && !need_report_nested_query()) { + return; + } + if (!query_desc->totaltime || !need_collect_analyze()) { + return; + } + // Make sure stats accumulation is done. + // (Note: it's okay if several levels of hook all do this.) + InstrEndLoop(query_desc->totaltime); + + double ms = query_desc->totaltime->total * 1000.0; + if (ms >= Config::min_analyze_time()) { + set_analyze_plan_text_json(query_desc, query_msg); + } +} + EventSender::EventSender() { if (Config::enable_collector()) { try { @@ -350,4 +393,4 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { EventSender::QueryItem::QueryItem(EventSender::QueryState st, yagpcc::SetQueryReq *msg) - : state(st), message(msg) {} \ No newline at end of file + : state(st), message(msg) {} diff --git a/src/EventSender.h b/src/EventSender.h index 99f7b24753d..4d09b429fc8 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -27,6 +27,7 @@ class EventSender { void executor_end(QueryDesc *query_desc); void query_metrics_collect(QueryMetricsStatus status, void *arg); void ic_metrics_collect(); + void analyze_stats_collect(QueryDesc *query_desc); void incr_depth() { nesting_level++; } void decr_depth() { nesting_level--; } EventSender(); diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 5982ff77c1c..ed3e69c6d44 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -109,3 +109,40 @@ ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { ExplainEndOutput(&es); return es; } + +ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze) { + ExplainState es; + ExplainInitState(&es); + es.analyze = analyze; + es.verbose = true; + es.buffers = es.analyze; + es.timing = es.analyze; + es.summary = es.analyze; + es.format = EXPLAIN_FORMAT_JSON; + ExplainBeginOutput(&es); + if (analyze) { + PG_TRY(); + { + ExplainPrintPlan(&es, query_desc); + ExplainPrintExecStatsEnd(&es, query_desc); + } + PG_CATCH(); + { + // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE + // implementation. We don't want any queries to fail due to those bugs, so + // we report the bug here for future investigatin and continue collecting + // metrics w/o reporting any plans + resetStringInfo(es.str); + appendStringInfo( + es.str, + "Unable to restore analyze plan due to PostgreSQL internal error. " + "See logs for more information"); + ereport(INFO, + (errmsg("YAGPCC failed to reconstruct analyze text for query: %s", + query_desc->sourceText))); + } + PG_END_TRY(); + } + ExplainEndOutput(&es); + return es; +} diff --git a/src/PgUtils.h b/src/PgUtils.h index 85b1eb833cd..81282a473a8 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -14,3 +14,4 @@ bool need_report_nested_query(); bool filter_query(QueryDesc *query_desc); bool need_collect(QueryDesc *query_desc, int nesting_level); ExplainState get_explain_state(QueryDesc *query_desc, bool costs); +ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index c37cefb72d6..6e9fa6bd5c5 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -8,6 +8,7 @@ extern "C" { #include "postgres.h" #include "access/hash.h" +#include "access/xact.h" #include "cdb/cdbinterconnect.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" @@ -47,8 +48,9 @@ void set_segment_key(yagpcc::SegmentKey *key) { key->set_segindex(GpIdentity.segindex); } -inline std::string char_to_trimmed_str(const char *str, size_t len) { - return std::string(str, std::min(len, Config::max_text_size())); +inline std::string char_to_trimmed_str(const char *str, size_t len, + size_t lim) { + return std::string(str, std::min(len, lim)); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -61,10 +63,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { MemoryContextSwitchTo(query_desc->estate->es_query_cxt); auto es = get_explain_state(query_desc, true); MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len); + *qi->mutable_plan_text() = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = - char_to_trimmed_str(norm_plan->data, norm_plan->len); + *qi->mutable_template_plan_text() = char_to_trimmed_str( + norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); pfree(es.str->data); @@ -76,10 +79,11 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = char_to_trimmed_str( - query_desc->sourceText, strlen(query_desc->sourceText)); + query_desc->sourceText, strlen(query_desc->sourceText), + Config::max_text_size()); char *norm_query = gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = - char_to_trimmed_str(norm_query, strlen(norm_query)); + *qi->mutable_template_query_text() = char_to_trimmed_str( + norm_query, strlen(norm_query), Config::max_text_size()); } } @@ -90,6 +94,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { qi->clear_template_plan_text(); qi->clear_query_text(); qi->clear_template_query_text(); + qi->clear_analyze_text(); } } @@ -115,7 +120,8 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { void set_qi_error_message(yagpcc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); auto error = elog_message(); - *aqi->mutable_error_message() = char_to_trimmed_str(error, strlen(error)); + *aqi->mutable_error_message() = + char_to_trimmed_str(error, strlen(error), Config::max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -217,4 +223,33 @@ yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +} + +void set_analyze_plan_text_json(QueryDesc *query_desc, + yagpcc::SetQueryReq *req) { + // Make sure it is a valid txn and it is not an utility + // statement for ExplainPrintPlan() later. + if (!IsTransactionState() || !query_desc->plannedstmt) { + return; + } + MemoryContext oldcxt = + MemoryContextSwitchTo(query_desc->estate->es_query_cxt); + + ExplainState es = get_analyze_state_json( + query_desc, query_desc->instrument_options && Config::enable_analyze()); + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { + es.str->data[--es.str->len] = '\0'; + } + // Convert JSON array to JSON object. + if (es.str->len > 0) { + es.str->data[0] = '{'; + es.str->data[es.str->len - 1] = '}'; + } + auto trimmed_analyze = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + + pfree(es.str->data); + MemoryContextSwitchTo(oldcxt); } \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 4e4ed5e76a3..6fb880c2eb8 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -16,4 +16,6 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); -double protots_to_double(const google::protobuf::Timestamp &ts); \ No newline at end of file +double protots_to_double(const google::protobuf::Timestamp &ts); +void set_analyze_plan_text_json(QueryDesc *query_desc, + yagpcc::SetQueryReq *message); \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index f1d403b82f1..79d3ec45881 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -3,6 +3,7 @@ extern "C" { #include "postgres.h" #include "funcapi.h" #include "executor/executor.h" +#include "executor/execUtils.h" #include "utils/elog.h" #include "utils/builtins.h" #include "utils/metrics_utils.h" @@ -24,6 +25,10 @@ static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; static ExecutorFinish_hook_type previous_ExecutorFinish_hook = nullptr; static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; +#ifdef ANALYZE_STATS_COLLECT_HOOK +static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = + nullptr; +#endif #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif @@ -36,6 +41,9 @@ static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); static void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); +#ifdef ANALYZE_STATS_COLLECT_HOOK +static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); +#endif static EventSender *sender = nullptr; @@ -71,6 +79,10 @@ void hooks_init() { #ifdef IC_TEARDOWN_HOOK previous_ic_teardown_hook = ic_teardown_hook; ic_teardown_hook = ya_ic_teardown_hook; +#endif +#ifdef ANALYZE_STATS_COLLECT_HOOK + previous_analyze_stats_collect_hook = analyze_stats_collect_hook; + analyze_stats_collect_hook = ya_analyze_stats_collect_hook; #endif stat_statements_parser_init(); } @@ -83,6 +95,9 @@ void hooks_deinit() { query_info_collect_hook = previous_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK ic_teardown_hook = previous_ic_teardown_hook; +#endif +#ifdef ANALYZE_STATS_COLLECT_HOOK + analyze_stats_collect_hook = previous_analyze_stats_collect_hook; #endif stat_statements_parser_deinit(); if (sender) { @@ -165,6 +180,15 @@ void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { #endif } +#ifdef ANALYZE_STATS_COLLECT_HOOK +void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { + cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); + if (previous_analyze_stats_collect_hook) { + (*previous_analyze_stats_collect_hook)(query_desc); + } +} +#endif + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), From 6e682428ef9f86ed861c9d1e85fae009097f5b9d Mon Sep 17 00:00:00 2001 From: NJrslv Date: Fri, 27 Jun 2025 13:31:34 +0300 Subject: [PATCH 089/167] [yagp_hooks_collector] Fix memory leaks, add safe C++ wrappers, improve Makefile Fix memory leaks in C++ and PG contexts. Add safe C++ wrappers around PG functions. Improve error message logging. Enable parallel make. Fix variable expansion. --- Makefile | 2 + src/Config.cpp | 22 +-- src/Config.h | 2 +- src/EventSender.cpp | 20 +-- src/EventSender.h | 3 - src/PgUtils.cpp | 106 +++---------- src/PgUtils.h | 6 +- src/ProcStats.cpp | 8 +- src/ProtoUtils.cpp | 73 ++++----- src/ProtoUtils.h | 2 + src/UDSConnector.cpp | 6 +- src/UDSConnector.h | 1 - src/hook_wrappers.cpp | 6 +- src/memory/gpdbwrappers.cpp | 148 ++++++++++++++++++ src/memory/gpdbwrappers.h | 131 ++++++++++++++++ .../pg_stat_statements_ya_parser.h | 6 +- 16 files changed, 380 insertions(+), 162 deletions(-) create mode 100644 src/memory/gpdbwrappers.cpp create mode 100644 src/memory/gpdbwrappers.h diff --git a/Makefile b/Makefile index 91be52c4468..15c5dabb70e 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. + # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. +all: all check install installdirs installcheck installcheck-parallel uninstall clean distclean maintainer-clean dist distcheck world check-world install-world installcheck-world installcheck-resgroup installcheck-resgroup-v2: @if [ ! -f GNUmakefile ] ; then \ diff --git a/src/Config.cpp b/src/Config.cpp index ac274a1e218..a1289a48891 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,5 @@ #include "Config.h" +#include "memory/gpdbwrappers.h" #include #include #include @@ -6,7 +7,6 @@ extern "C" { #include "postgres.h" -#include "utils/builtins.h" #include "utils/guc.h" } @@ -29,15 +29,15 @@ static void update_ignored_users(const char *new_guc_ignored_users) { std::make_unique>(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = pstrdup(new_guc_ignored_users); + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!SplitIdentifierString(rawstring, ',', &elemlist)) { + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - pfree(rawstring); - list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), @@ -48,8 +48,8 @@ static void update_ignored_users(const char *new_guc_ignored_users) { foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - pfree(rawstring); - list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); } ignored_users_set = std::move(new_ignored_users_set); } @@ -119,11 +119,11 @@ size_t Config::max_text_size() { return guc_max_text_size * 1024; } size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } int Config::min_analyze_time() { return guc_min_analyze_time; }; -bool Config::filter_user(const std::string *username) { - if (!username || !ignored_users_set) { +bool Config::filter_user(std::string username) { + if (!ignored_users_set) { return true; } - return ignored_users_set->find(*username) != ignored_users_set->end(); + return ignored_users_set->find(username) != ignored_users_set->end(); } void Config::sync() { @@ -131,4 +131,4 @@ void Config::sync() { update_ignored_users(guc_ignored_users); ignored_users_guc_dirty = false; } -} \ No newline at end of file +} diff --git a/src/Config.h b/src/Config.h index dd081c41dd6..eff83f0960a 100644 --- a/src/Config.h +++ b/src/Config.h @@ -9,7 +9,7 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); - static bool filter_user(const std::string *username); + static bool filter_user(std::string username); static bool report_nested_queries(); static size_t max_text_size(); static size_t max_plan_size(); diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 19787fe0db0..8711c4cbd4f 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,15 +1,14 @@ #include "Config.h" #include "UDSConnector.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid extern "C" { #include "postgres.h" -#include "access/hash.h" #include "executor/executor.h" #include "utils/elog.h" -#include "cdb/cdbdisp.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" @@ -81,7 +80,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - cdbexplain_showExecStatsBegin(query_desc, starttime); + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -106,10 +105,10 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { // Make sure the space is allocated in the per-query // context so it will go away at executor_end. if (query_desc->totaltime == NULL) { - MemoryContext oldcxt; - oldcxt = MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - query_desc->totaltime = InstrAlloc(1, INSTRUMENT_ALL); - MemoryContextSwitchTo(oldcxt); + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL); + gpdb::mem_ctx_switch_to(oldcxt); } } yagpcc::GPMetrics stats; @@ -240,7 +239,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}); - pfree(query_desc->gpmon_pkt); + gpdb::pfree(query_desc->gpmon_pkt); } } @@ -297,7 +296,7 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - InstrEndLoop(query_desc->totaltime); + gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { @@ -364,7 +363,8 @@ EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, query_desc->gpmon_pkt->u.qexec.key.tmid}) == query_msgs.end()) { - query_desc->gpmon_pkt = (gpmon_packet_t *)palloc0(sizeof(gpmon_packet_t)); + query_desc->gpmon_pkt = + (gpmon_packet_t *)gpdb::palloc0(sizeof(gpmon_packet_t)); query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; query_msgs.insert({{gp_command_count, nesting_level}, diff --git a/src/EventSender.h b/src/EventSender.h index 4d09b429fc8..f3dd1d2a528 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,13 +1,10 @@ #pragma once -#include #include -#include #define typeid __typeid extern "C" { #include "utils/metrics_utils.h" -#include "cdb/ml_ipc.h" #ifdef IC_TEARDOWN_HOOK #include "cdb/ic_udpifc.h" #endif diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index ed3e69c6d44..f36cd030a39 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -1,37 +1,41 @@ #include "PgUtils.h" #include "Config.h" +#include "memory/gpdbwrappers.h" extern "C" { -#include "utils/guc.h" -#include "commands/dbcommands.h" #include "commands/resgroupcmds.h" #include "cdb/cdbvars.h" } -std::string *get_user_name() { - const char *username = GetConfigOption("session_authorization", false, false); - // username is not to be freed - return username ? new std::string(username) : nullptr; +std::string get_user_name() { + // username is allocated on stack, we don't need to pfree it. + const char *username = + ya_gpdb::get_config_option("session_authorization", false, false); + return username ? std::string(username) : ""; } -std::string *get_db_name() { - char *dbname = get_database_name(MyDatabaseId); - std::string *result = nullptr; +std::string get_db_name() { + char *dbname = ya_gpdb::get_database_name(MyDatabaseId); if (dbname) { - result = new std::string(dbname); - pfree(dbname); + std::string result(dbname); + ya_gpdb::pfree(dbname); + return result; } - return result; + return ""; } -std::string *get_rg_name() { - auto groupId = ResGroupGetGroupIdBySessionId(MySessionState->sessionId); +std::string get_rg_name() { + auto groupId = ya_gpdb::get_rg_id_by_session_id(MySessionState->sessionId); if (!OidIsValid(groupId)) - return nullptr; - char *rgname = GetResGroupNameForId(groupId); + return ""; + + char *rgname = ya_gpdb::get_rg_name_for_id(groupId); if (rgname == nullptr) - return nullptr; - return new std::string(rgname); + return ""; + + std::string result(rgname); + ya_gpdb::pfree(rgname); + return result; } /** @@ -80,69 +84,3 @@ bool need_collect(QueryDesc *query_desc, int nesting_level) { return !filter_query(query_desc) && nesting_is_valid(query_desc, nesting_level); } - -ExplainState get_explain_state(QueryDesc *query_desc, bool costs) { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - PG_TRY(); - { ExplainPrintPlan(&es, query_desc); } - PG_CATCH(); - { - // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE - // implementation. We don't want any queries to fail due to those bugs, so - // we report the bug here for future investigatin and continue collecting - // metrics w/o reporting any plans - resetStringInfo(es.str); - appendStringInfo( - es.str, - "Unable to restore query plan due to PostgreSQL internal error. " - "See logs for more information"); - ereport(INFO, - (errmsg("YAGPCC failed to reconstruct explain text for query: %s", - query_desc->sourceText))); - } - PG_END_TRY(); - ExplainEndOutput(&es); - return es; -} - -ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze) { - ExplainState es; - ExplainInitState(&es); - es.analyze = analyze; - es.verbose = true; - es.buffers = es.analyze; - es.timing = es.analyze; - es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_JSON; - ExplainBeginOutput(&es); - if (analyze) { - PG_TRY(); - { - ExplainPrintPlan(&es, query_desc); - ExplainPrintExecStatsEnd(&es, query_desc); - } - PG_CATCH(); - { - // PG and GP both have known and yet unknown bugs in EXPLAIN VERBOSE - // implementation. We don't want any queries to fail due to those bugs, so - // we report the bug here for future investigatin and continue collecting - // metrics w/o reporting any plans - resetStringInfo(es.str); - appendStringInfo( - es.str, - "Unable to restore analyze plan due to PostgreSQL internal error. " - "See logs for more information"); - ereport(INFO, - (errmsg("YAGPCC failed to reconstruct analyze text for query: %s", - query_desc->sourceText))); - } - PG_END_TRY(); - } - ExplainEndOutput(&es); - return es; -} diff --git a/src/PgUtils.h b/src/PgUtils.h index 81282a473a8..ceb07c2e8e5 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -5,9 +5,9 @@ extern "C" { #include -std::string *get_user_name(); -std::string *get_db_name(); -std::string *get_rg_name(); +std::string get_user_name(); +std::string get_db_name(); +std::string get_rg_name(); bool is_top_level_query(QueryDesc *query_desc, int nesting_level); bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); bool need_report_nested_query(); diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index a557a20cbb0..5c09fa0bce4 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -75,16 +75,16 @@ void fill_status_stats(yagpcc::SystemStat *stats) { stats->set_vmpeakkb(value); proc_stat >> measure; if (measure != "kB") { - ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", - measure.c_str()))); + throw std::runtime_error("Expected memory sizes in kB, but got in " + + measure); } } else if (key == "VmSize:") { uint64_t value; proc_stat >> value; stats->set_vmsizekb(value); if (measure != "kB") { - ereport(FATAL, (errmsg("Expected memory sizes in kB, but got in %s", - measure.c_str()))); + throw std::runtime_error("Expected memory sizes in kB, but got in " + + measure); } } } diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 6e9fa6bd5c5..6dc39278bcd 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -2,6 +2,7 @@ #include "PgUtils.h" #include "ProcStats.h" #include "Config.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid #define operator __operator @@ -15,10 +16,7 @@ extern "C" { #ifdef IC_TEARDOWN_HOOK #include "cdb/ic_udpifc.h" #endif -#include "gpmon/gpmon.h" #include "utils/workfile_mgr.h" - -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" } #undef typeid #undef operator @@ -60,18 +58,21 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - auto es = get_explain_state(query_desc, true); - MemoryContextSwitchTo(oldcxt); - *qi->mutable_plan_text() = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - StringInfo norm_plan = gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = char_to_trimmed_str( - norm_plan->data, norm_plan->len, Config::max_plan_size()); - qi->set_plan_id(hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - qi->set_query_id(query_desc->plannedstmt->queryId); - pfree(es.str->data); - pfree(norm_plan->data); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); + if (es.str) { + *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, + Config::max_plan_size()); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + *qi->mutable_template_plan_text() = char_to_trimmed_str( + norm_plan->data, norm_plan->len, Config::max_plan_size()); + qi->set_plan_id( + hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + qi->set_query_id(query_desc->plannedstmt->queryId); + gpdb::pfree(es.str->data); + gpdb::pfree(norm_plan->data); + } + gpdb::mem_ctx_switch_to(oldcxt); } } @@ -81,7 +82,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { *qi->mutable_query_text() = char_to_trimmed_str( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); - char *norm_query = gen_normquery(query_desc->sourceText); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = char_to_trimmed_str( norm_query, strlen(norm_query), Config::max_text_size()); } @@ -101,9 +102,9 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { void set_query_info(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); - qi->set_allocated_username(get_user_name()); - qi->set_allocated_databasename(get_db_name()); - qi->set_allocated_rsgname(get_rg_name()); + qi->set_username(get_user_name()); + qi->set_databasename(get_db_name()); + qi->set_rsgname(get_rg_name()); } } @@ -233,23 +234,23 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, return; } MemoryContext oldcxt = - MemoryContextSwitchTo(query_desc->estate->es_query_cxt); - - ExplainState es = get_analyze_state_json( + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state_json( query_desc, query_desc->instrument_options && Config::enable_analyze()); - // Remove last line break. - if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { - es.str->data[--es.str->len] = '\0'; - } - // Convert JSON array to JSON object. - if (es.str->len > 0) { - es.str->data[0] = '{'; - es.str->data[es.str->len - 1] = '}'; + gpdb::mem_ctx_switch_to(oldcxt); + if (es.str) { + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { + es.str->data[--es.str->len] = '\0'; + } + // Convert JSON array to JSON object. + if (es.str->len > 0) { + es.str->data[0] = '{'; + es.str->data[es.str->len - 1] = '}'; + } + auto trimmed_analyze = + char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + gpdb::pfree(es.str->data); } - auto trimmed_analyze = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - req->mutable_query_info()->set_analyze_text(trimmed_analyze); - - pfree(es.str->data); - MemoryContextSwitchTo(oldcxt); } \ No newline at end of file diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 6fb880c2eb8..8287b3de7ea 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,3 +1,5 @@ +#pragma once + #include "protos/yagpcc_set_service.pb.h" struct QueryDesc; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 8a5f754f3b4..b5b70836db4 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,6 +1,7 @@ #include "UDSConnector.h" #include "Config.h" #include "YagpStat.h" +#include "memory/gpdbwrappers.h" #include #include @@ -13,7 +14,6 @@ extern "C" { #include "postgres.h" -#include "cdb/cdbvars.h" } UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } @@ -44,7 +44,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { auto data_size = req.ByteSize(); auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)palloc(total_size); + uint8_t *buf = (uint8_t *)gpdb::palloc(total_size); uint32_t *size_payload = (uint32_t *)buf; *size_payload = data_size; req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); @@ -67,7 +67,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, } else { YagpStat::report_send(total_size); } - pfree(buf); + gpdb::pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 42e0aa20968..67504fc8529 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -1,7 +1,6 @@ #pragma once #include "protos/yagpcc_set_service.pb.h" -#include class UDSConnector { public: diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 79d3ec45881..25a85f086d1 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -7,10 +7,10 @@ extern "C" { #include "utils/elog.h" #include "utils/builtins.h" #include "utils/metrics_utils.h" -#include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" #include "tcop/utility.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" } #undef typeid @@ -18,7 +18,7 @@ extern "C" { #include "YagpStat.h" #include "EventSender.h" #include "hook_wrappers.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "memory/gpdbwrappers.h" static ExecutorStart_hook_type previous_ExecutorStart_hook = nullptr; static ExecutorRun_hook_type previous_ExecutorRun_hook = nullptr; @@ -229,7 +229,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp new file mode 100644 index 00000000000..1fba702a9f5 --- /dev/null +++ b/src/memory/gpdbwrappers.cpp @@ -0,0 +1,148 @@ +#include "gpdbwrappers.h" + +extern "C" { +#include "postgres.h" +#include "utils/guc.h" +#include "commands/dbcommands.h" +#include "commands/resgroupcmds.h" +#include "utils/builtins.h" +#include "nodes/pg_list.h" +#include "commands/explain.h" +#include "executor/instrument.h" +#include "access/tupdesc.h" +#include "access/htup.h" +#include "utils/elog.h" +#include "cdb/cdbexplain.h" +#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +} + +void *gpdb::palloc(Size size) { return detail::wrap_throw(::palloc, size); } + +void *gpdb::palloc0(Size size) { return detail::wrap_throw(::palloc0, size); } + +char *gpdb::pstrdup(const char *str) { + return detail::wrap_throw(::pstrdup, str); +} + +char *gpdb::get_database_name(Oid dbid) noexcept { + return detail::wrap_noexcept(::get_database_name, dbid); +} + +bool gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept { + return detail::wrap_noexcept(SplitIdentifierString, rawstring, separator, + namelist); +} + +ExplainState gpdb::get_explain_state(QueryDesc *query_desc, + bool costs) noexcept { + return detail::wrap_noexcept([&]() { + ExplainState es; + ExplainInitState(&es); + es.costs = costs; + es.verbose = true; + es.format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(&es); + ExplainPrintPlan(&es, query_desc); + ExplainEndOutput(&es); + return es; + }); +} + +ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept { + return detail::wrap_noexcept([&]() { + ExplainState es; + ExplainInitState(&es); + es.analyze = analyze; + es.verbose = true; + es.buffers = es.analyze; + es.timing = es.analyze; + es.summary = es.analyze; + es.format = EXPLAIN_FORMAT_JSON; + ExplainBeginOutput(&es); + if (analyze) { + ExplainPrintPlan(&es, query_desc); + ExplainPrintExecStatsEnd(&es, query_desc); + } + ExplainEndOutput(&es); + return es; + }); +} + +Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options) { + return detail::wrap_throw(InstrAlloc, n, instrument_options); +} + +HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull) { + if (!tupleDescriptor || !values || !isnull) + throw std::runtime_error( + "Invalid input parameters for heap tuple formation"); + + return detail::wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); +} + +void gpdb::pfree(void *pointer) noexcept { + // Note that ::pfree asserts that pointer != NULL. + if (!pointer) + return; + + detail::wrap_noexcept(::pfree, pointer); +} + +MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { + return MemoryContextSwitchTo(context); +} + +const char *gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept { + if (!name) + return nullptr; + + return detail::wrap_noexcept(GetConfigOption, name, missing_ok, + restrict_superuser); +} + +void gpdb::list_free(List *list) noexcept { + if (!list) + return; + + detail::wrap_noexcept(::list_free, list); +} + +CdbExplain_ShowStatCtx * +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime) { + if (!query_desc) + throw std::runtime_error("Invalid query descriptor"); + + return detail::wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, + starttime); +} + +void gpdb::instr_end_loop(Instrumentation *instr) { + if (!instr) + throw std::runtime_error("Invalid instrumentation pointer"); + + detail::wrap_throw(::InstrEndLoop, instr); +} + +char *gpdb::gen_normquery(const char *query) { + return detail::wrap_throw(::gen_normquery, query); +} + +StringInfo gpdb::gen_normplan(const char *exec_plan) { + if (!exec_plan) + throw std::runtime_error("Invalid execution plan string"); + + return detail::wrap_throw(::gen_normplan, exec_plan); +} + +char *gpdb::get_rg_name_for_id(Oid group_id) { + return detail::wrap_throw(GetResGroupNameForId, group_id); +} + +Oid gpdb::get_rg_id_by_session_id(int session_id) { + return detail::wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +} \ No newline at end of file diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h new file mode 100644 index 00000000000..437a5dd5d29 --- /dev/null +++ b/src/memory/gpdbwrappers.h @@ -0,0 +1,131 @@ +#pragma once + +extern "C" { +#include "postgres.h" +#include "nodes/pg_list.h" +#include "commands/explain.h" +#include "executor/instrument.h" +#include "access/htup.h" +#include "utils/elog.h" +#include "utils/memutils.h" +} + +#include +#include +#include +#include +#include + +namespace gpdb { +namespace detail { + +template +auto wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) { + + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult {}; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) { + result_holder.emplace(func(std::forward(args)...)); + } else { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) { + std::string err; + if (edata && edata->message) { + err = std::string(edata->message); + } else { + err = "Unknown error occurred"; + } + + if (edata) { + FreeErrorData(edata); + } + + if constexpr (Throws) { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) { + return RetType{}; + } else { + return; + } + } + + if constexpr (!std::is_void_v) { + return *std::move(result_holder); + } else { + return; + } +} + +template +auto wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) { + return detail::wrap(std::forward(func), + std::forward(args)...); +} + +template +auto wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) { + return detail::wrap(std::forward(func), + std::forward(args)...); +} +} // namespace detail + +// Functions that call palloc(). +// Make sure correct memory context is set. +void *palloc(Size size); +void *palloc0(Size size); +char *pstrdup(const char *str); +char *get_database_name(Oid dbid) noexcept; +bool split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept; +ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; +ExplainState get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept; +Instrumentation *instr_alloc(size_t n, int instrument_options); +HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull); +CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime); +void instr_end_loop(Instrumentation *instr); +char *gen_normquery(const char *query); +StringInfo gen_normplan(const char *executionPlan); +char *get_rg_name_for_id(Oid group_id); + +// Palloc-free functions. +void pfree(void *pointer) noexcept; +MemoryContext mem_ctx_switch_to(MemoryContext context) noexcept; +const char *get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept; +void list_free(List *list) noexcept; +Oid get_rg_id_by_session_id(int session_id); + +} // namespace gpdb diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index aa9cd217e31..b08e8533992 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -8,9 +8,9 @@ extern "C" extern void stat_statements_parser_init(void); extern void stat_statements_parser_deinit(void); +StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query); + #ifdef __cplusplus } #endif - -StringInfo gen_normplan(const char *executionPlan); -char *gen_normquery(const char *query); \ No newline at end of file From effbd4bb70fe910d3d6f04663f4a983fa261ef1e Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 14 Jul 2025 16:14:49 +0300 Subject: [PATCH 090/167] [yagp_hooks_collector] Add utility statement tracking and metrics documentation Hook into ProcessUtility to emit submit and done events for DDL. Add metrics documentation (metric.md). Change namespace to avoid GPOS conflicts. Report incomplete queries at extension shutdown. Clean up stray files. --- metric.md | 125 +++++++++++ src/Config.cpp | 12 +- src/EventSender.cpp | 407 ++++++++++++++++++++---------------- src/EventSender.h | 94 +++++++-- src/PgUtils.cpp | 10 +- src/ProtoUtils.cpp | 22 +- src/UDSConnector.cpp | 4 +- src/hook_wrappers.cpp | 2 +- src/memory/gpdbwrappers.cpp | 163 +++++++++++---- src/memory/gpdbwrappers.h | 85 +------- 10 files changed, 572 insertions(+), 352 deletions(-) create mode 100644 metric.md diff --git a/metric.md b/metric.md new file mode 100644 index 00000000000..2d198391a67 --- /dev/null +++ b/metric.md @@ -0,0 +1,125 @@ +## YAGP Hooks Collector Metrics + +### States +A Postgres process goes through 4 executor functions to execute a query: +1) `ExecutorStart()` - resource allocation for the query. +2) `ExecutorRun()` - query execution. +3) `ExecutorFinish()` - cleanup. +4) `ExecutorEnd()` - cleanup. + +yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +``` +submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done +``` + +### Key Points +- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. +- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. +- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. +- Nested queries on _Dispatcher_ become top-level on _Execute_. +- Each process (_Dispatcher_/_Execute_) sends its own statistics. + +### Notations +- **S** = Submit event. +- **T** = Start event. +- **E** = End event. +- **D** = Done event. +- **DIFF** = current_value - submit_value (submit event). +- **ABS** = Absolute value, or where diff is not applicable, the value taken. +- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. + +### Statistics Table + +| Proto Field | Type | When | DIFF/ABS | Local* | Scope | Dispatcher | Execute | Units | Notes | +| :--------------------------- | :----- | :------ | :------- | ------ | :------ | :--------: | :-----: | :------ | :-------------------------------------------------- | +| **SystemStat** | | | | | | | | | | +| `runningTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | Wall clock time | +| `userTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat utime | +| `kernelTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat stime | +| `vsize` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat vsize | +| `rss` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat rss | +| `VmSizeKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmSize | +| `VmPeakKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmPeak | +| `rchar` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io rchar | +| `wchar` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io wchar | +| `syscr` | uint64 | E, D | DIFF | - | Node | + | + | count | /proc/pid/io syscr | +| `syscw` | uint64 | E, D | DIFF | - | Node | + | + | count | /proc/pid/io syscw | +| `read_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io read_bytes | +| `write_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io write_bytes | +| `cancelled_write_bytes` | uint64 | E, D | DIFF | - | Node | + | + | bytes | /proc/pid/io cancelled_write_bytes | +| **MetricInstrumentation** | | | | | | | | | | +| `ntuples` | uint64 | E, D | ABS | + | Node | + | + | tuples | Accumulated total tuples | +| `nloops` | uint64 | E, D | ABS | + | Node | + | + | count | Number of cycles | +| `tuplecount` | uint64 | E, D | ABS | + | Node | + | + | tuples | Accumulated tuples per cycle | +| `firsttuple` | double | E, D | ABS | + | Node | + | + | seconds | Time for first tuple of this cycle | +| `startup` | double | E, D | ABS | + | Node | + | + | seconds | Start time of current iteration | +| `total` | double | E, D | ABS | + | Node | + | + | seconds | Total time taken | +| `shared_blks_hit` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared buffer blocks found in cache | +| `shared_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared buffer blocks read from disk | +| `shared_blks_dirtied` | uint64 | E, D | ABS | + | Node | + | + | blocks | Shared blocks dirtied | +| `shared_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Dirty shared buffer blocks written to disk | +| `local_blks_hit` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local buffer hits | +| `local_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Disk blocks read | +| `local_blks_dirtied` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local blocks dirtied | +| `local_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Local blocks written to disk | +| `temp_blks_read` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks read | +| `temp_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks written | +| `blk_read_time` | double | E, D | ABS | + | Node | + | + | seconds | Time reading data blocks | +| `blk_write_time` | double | E, D | ABS | + | Node | + | + | seconds | Time writing data blocks | +| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (YAGPCC-specific) | +| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (YAGPCC-specific) | +| **NetworkStat (sent)** | | | | | | | | | | +| `sent.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes sent, including headers | +| `sent.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data sent | +| `sent.chunks` | uint32 | D | ABS | - | Node | + | + | count | Tuple-chunks sent | +| **NetworkStat (received)** | | | | | | | | | | +| `received.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data received | +| `received.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data received | +| `received.chunks` | uint32 | D | ABS | - | Node | + | + | count | Tuple-chunks received | +| **InterconnectStat** | | | | | | | | | | +| `total_recv_queue_size` | uint64 | D | DIFF | - | Node | + | + | bytes | Receive queue size sum | +| `recv_queue_size_counting_t` | uint64 | D | DIFF | - | Node | + | + | count | Counting times when computing total_recv_queue_size | +| `total_capacity` | uint64 | D | DIFF | - | Node | + | + | bytes | the capacity sum for sent packets | +| `capacity_counting_time` | uint64 | D | DIFF | - | Node | + | + | count | counting times used to compute total_capacity | +| `total_buffers` | uint64 | D | DIFF | - | Node | + | + | count | Available buffers | +| `buffer_counting_time` | uint64 | D | DIFF | - | Node | + | + | count | counting times when compute total_buffers | +| `active_connections_num` | uint64 | D | DIFF | - | Node | + | + | count | Active connections | +| `retransmits` | int64 | D | DIFF | - | Node | + | + | count | Packet retransmits | +| `startup_cached_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Startup cached packets | +| `mismatch_num` | int64 | D | DIFF | - | Node | + | + | count | Mismatched packets received | +| `crc_errors` | int64 | D | DIFF | - | Node | + | + | count | CRC errors | +| `snd_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Packets sent | +| `recv_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Packets received | +| `disordered_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Out-of-order packets | +| `duplicated_pkt_num` | int64 | D | DIFF | - | Node | + | + | count | Duplicate packets | +| `recv_ack_num` | int64 | D | DIFF | - | Node | + | + | count | ACKs received | +| `status_query_msg_num` | int64 | D | DIFF | - | Node | + | + | count | Status query messages sent | +| **SpillInfo** | | | | | | | | | | +| `fileCount` | int32 | E, D | DIFF | - | Node | + | + | count | Spill (temp) files created | +| `totalBytes` | int64 | E, D | DIFF | - | Node | + | + | bytes | Spill bytes written | +| **QueryInfo** | | | | | | | | | | +| `generator` | enum | T, E, D | ABS | - | Cluster | + | - | enum | Planner/Optimizer | +| `query_id` | uint64 | T, E, D | ABS | - | Cluster | + | - | id | Query ID | +| `plan_id` | uint64 | T, E, D | ABS | - | Cluster | + | - | id | Hash of normalized plan | +| `query_text` | string | S | ABS | - | Cluster | + | - | text | Query text | +| `plan_text` | string | T | ABS | - | Cluster | + | - | text | EXPLAIN text | +| `template_query_text` | string | S | ABS | - | Cluster | + | - | text | Normalized query text | +| `template_plan_text` | string | T | ABS | - | Cluster | + | - | text | Normalized plan text | +| `userName` | string | All | ABS | - | Cluster | + | - | text | Session user | +| `databaseName` | string | All | ABS | - | Cluster | + | - | text | Database name | +| `rsgname` | string | All | ABS | - | Cluster | + | - | text | Resource group name | +| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE JSON | +| **AdditionalQueryInfo** | | | | | | | | | | +| `nested_level` | int64 | All | ABS | - | Node | + | + | count | Current nesting level | +| `error_message` | string | D | ABS | - | Node | + | + | text | Error message | +| `slice_id` | int64 | All | ABS | - | Node | + | + | id | Slice ID | +| **QueryKey** | | | | | | | | | | +| `tmid` | int32 | All | ABS | - | Node | + | + | id | Time ID | +| `ssid` | int32 | All | ABS | - | Node | + | + | id | Session ID | +| `ccnt` | int32 | All | ABS | - | Node | + | + | count | Command counter | +| **SegmentKey** | | | | | | | | | | +| `dbid` | int32 | All | ABS | - | Node | + | + | id | Database ID | +| `segment_index` | int32 | All | ABS | - | Node | + | + | id | Segment index (-1=coordinator) | + +--- + diff --git a/src/Config.cpp b/src/Config.cpp index a1289a48891..aef09fc7d73 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -29,15 +29,15 @@ static void update_ignored_users(const char *new_guc_ignored_users) { std::make_unique>(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = gpdb::pstrdup(new_guc_ignored_users); + char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), @@ -48,8 +48,8 @@ static void update_ignored_users(const char *new_guc_ignored_users) { foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); } ignored_users_set = std::move(new_ignored_users_set); } diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 8711c4cbd4f..133d409b574 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -8,6 +8,7 @@ extern "C" { #include "executor/executor.h" #include "utils/elog.h" +#include "utils/guc.h" #include "cdb/cdbexplain.h" #include "cdb/cdbvars.h" @@ -27,6 +28,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { return; } + auto *query_desc = reinterpret_cast(arg); switch (status) { case METRICS_PLAN_NODE_INITIALIZE: case METRICS_PLAN_NODE_EXECUTING: @@ -34,8 +36,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - // don't collect anything here. We will fake this call in ExecutorStart as - // it really makes no difference. Just complicates things + collect_query_submit(query_desc); break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -49,7 +50,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: - collect_query_done(reinterpret_cast(arg), status); + collect_query_done(query_desc, status); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -60,15 +61,15 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { if (!connector) { return; } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; + if (filter_query(query_desc)) { + return; + } + if (!qdesc_submitted(query_desc)) { + collect_query_submit(query_desc); } - Config::sync(); if (!need_collect(query_desc, nesting_level)) { return; } - collect_query_submit(query_desc); if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; @@ -80,167 +81,194 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + ya_gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!connector) { + if (!connector || !need_collect(query_desc, nesting_level)) { return; } - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { - if (!filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - auto query_msg = query->message; - *query_msg->mutable_start_time() = current_ts(); - if (!nesting_is_valid(query_desc, nesting_level)) { - return; - } - update_query_state(query_desc, query, QueryState::START); - set_query_plan(query_msg, query_desc); - if (need_collect_analyze()) { - // Set up to track total elapsed time during query run. - // Make sure the space is allocated in the per-query - // context so it will go away at executor_end. - if (query_desc->totaltime == NULL) { - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL); - gpdb::mem_ctx_switch_to(oldcxt); - } - } - yagpcc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { - clear_big_fields(query_msg); - } - std::swap(stats, *query_msg->mutable_query_metrics()); + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto &query = get_query(query_desc); + auto query_msg = query.message.get(); + *query_msg->mutable_start_time() = current_ts(); + update_query_state(query, QueryState::START); + set_query_plan(query_msg, query_desc); + if (need_collect_analyze()) { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) { + MemoryContext oldcxt = + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL); + ya_gpdb::mem_ctx_switch_to(oldcxt); } } + yagpcc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (connector->report_query(*query_msg, "started")) { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector || - (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE)) { + if (!connector || !need_collect(query_desc, nesting_level)) { return; } - if (!filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - auto query_msg = query->message; - *query_msg->mutable_end_time() = current_ts(); - if (nesting_is_valid(query_desc, nesting_level)) { - if (query->state == UNKNOWN && - // Yet another greenplum weirdness: thats actually a nested query - // which is being committed/rollbacked. Treat it accordingly. - !need_report_nested_query()) { - return; - } - update_query_state(query_desc, query, QueryState::END); - if (is_top_level_query(query_desc, nesting_level)) { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, - nested_calls, nested_timing); - } else { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); - } - if (connector->report_query(*query_msg, "ended")) { - clear_big_fields(query_msg); - } - } + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + return; + } + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg->mutable_end_time() = current_ts(); + update_query_state(query, QueryState::END); + if (is_top_level_query(query_desc, nesting_level)) { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, + nested_timing); + } else { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (connector->report_query(*query_msg, "ended")) { + clear_big_fields(query_msg); } } void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (connector && need_collect(query_desc, nesting_level)) { - auto *query = get_query_message(query_desc); - query->state = QueryState::SUBMIT; - auto query_msg = query->message; - *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); - *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg); - set_qi_nesting_level(query_msg, query_desc->gpmon_pkt->u.qexec.key.tmid); - set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc); - if (connector->report_query(*query_msg, "submit")) { - clear_big_fields(query_msg); - } - // take initial metrics snapshot so that we can safely take diff afterwards - // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + if (!connector) { + return; + } + Config::sync(); + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) { + QueryKey::register_qkey(query_desc, nesting_level); + return; + } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } + if (!need_collect(query_desc, nesting_level)) { + return; + } + submit_query(query_desc); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg); + set_qi_nesting_level(query_msg, nesting_level); + set_qi_slice_id(query_msg); + set_query_text(query_msg, query_desc); + if (connector->report_query(*query_msg, "submit")) { + clear_big_fields(query_msg); + } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); #ifdef IC_TEARDOWN_HOOK - // same for interconnect statistics - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif +} + +void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status) { + yagpcc::QueryStatus query_status; + std::string msg; + switch (status) { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(FATAL, + (errmsg("Unexpected query status in query_done hook: %d", status))); } + auto prev_state = query.state; + update_query_state(query, QueryState::DONE, + query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query.message.get(); + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) { + set_qi_error_message(query_msg); + } + if (prev_state == START) { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, + nested_timing); + } +#ifdef IC_TEARDOWN_HOOK + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); +#endif + connector->report_query(*query_msg, msg); } void EventSender::collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status) { - if (connector && !filter_query(query_desc)) { - auto *query = get_query_message(query_desc); - if (query->state != UNKNOWN || need_report_nested_query()) { - if (nesting_is_valid(query_desc, nesting_level)) { - yagpcc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code - // here just in case - Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, - (errmsg("Unexpected query status in query_done hook: %d", - status))); - } - auto prev_state = query->state; - update_query_state(query_desc, query, QueryState::DONE, - query_status == - yagpcc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query->message; - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); - } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, - nested_calls, nested_timing); - } -#ifdef IC_TEARDOWN_HOOK - ic_metrics_collect(); - set_ic_stats( - query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); -#endif - connector->report_query(*query_msg, msg); - } - update_nested_counters(query_desc); + if (!connector || !need_collect(query_desc, nesting_level)) { + return; + } + + // Skip sending done message if query errored before submit. + if (!qdesc_submitted(query_desc)) { + if (status != METRICS_QUERY_ERROR) { + ereport(WARNING, (errmsg("YAGPCC trying to process DONE hook for " + "unsubmitted and unerrored query"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); } - query_msgs.erase({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}); - gpdb::pfree(query_desc->gpmon_pkt); + return; + } + + if (queries.empty()) { + ereport(WARNING, (errmsg("YAGPCC cannot find query to process DONE hook"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + return; } + auto &query = get_query(query_desc); + + bool report = need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); + if (report) + report_query_done(query_desc, query, status); + + if (need_report_nested_query()) + update_nested_counters(query_desc); + + queries.erase(QueryKey::from_qdesc(query_desc)); + pfree(query_desc->yagp_query_key); + query_desc->yagp_query_key = NULL; } void EventSender::ic_metrics_collect() { @@ -283,20 +311,15 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { if (!need_collect(query_desc, nesting_level)) { return; } - auto query = get_query_message(query_desc); - auto query_msg = query->message; + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); *query_msg->mutable_end_time() = current_ts(); - // Yet another greenplum weirdness: thats actually a nested query - // which is being committed/rollbacked. Treat it accordingly. - if (query->state == UNKNOWN && !need_report_nested_query()) { - return; - } if (!query_desc->totaltime || !need_collect_analyze()) { return; } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - gpdb::instr_end_loop(query_desc->totaltime); + ya_gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { @@ -318,26 +341,26 @@ EventSender::EventSender() { } EventSender::~EventSender() { - delete connector; - for (auto iter = query_msgs.begin(); iter != query_msgs.end(); ++iter) { - delete iter->second.message; + for (const auto &[qkey, _] : queries) { + ereport(LOG, + (errmsg("YAGPCC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } + delete connector; } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP -void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, - QueryState new_state, bool success) { - if (query->state == UNKNOWN) { - collect_query_submit(query_desc); - } +void EventSender::update_query_state(QueryItem &query, QueryState new_state, + bool success) { switch (new_state) { case QueryState::SUBMIT: Assert(false); break; case QueryState::START: - if (query->state == QueryState::SUBMIT) { - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + if (query.state == QueryState::SUBMIT) { + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); } else { Assert(false); } @@ -346,40 +369,52 @@ void EventSender::update_query_state(QueryDesc *query_desc, QueryItem *query, // Example of below assert triggering: CURSOR closes before ever being // executed Assert(query->state == QueryState::START || // IsAbortInProgress()); - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: - Assert(query->state == QueryState::END || !success); - query->message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + Assert(query.state == QueryState::END || !success); + query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); break; default: Assert(false); } - query->state = new_state; + query.state = new_state; } -EventSender::QueryItem *EventSender::get_query_message(QueryDesc *query_desc) { - if (query_desc->gpmon_pkt == nullptr || - query_msgs.find({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}) == - query_msgs.end()) { - query_desc->gpmon_pkt = - (gpmon_packet_t *)gpdb::palloc0(sizeof(gpmon_packet_t)); - query_desc->gpmon_pkt->u.qexec.key.ccnt = gp_command_count; - query_desc->gpmon_pkt->u.qexec.key.tmid = nesting_level; - query_msgs.insert({{gp_command_count, nesting_level}, - QueryItem(UNKNOWN, new yagpcc::SetQueryReq())}); - } - return &query_msgs.at({query_desc->gpmon_pkt->u.qexec.key.ccnt, - query_desc->gpmon_pkt->u.qexec.key.tmid}); +EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { + if (!qdesc_submitted(query_desc)) { + ereport(WARNING, + (errmsg("YAGPCC attempting to get query that was not submitted"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + throw std::runtime_error("Attempting to get query that was not submitted"); + } + return queries.find(QueryKey::from_qdesc(query_desc))->second; +} + +void EventSender::submit_query(QueryDesc *query_desc) { + if (query_desc->yagp_query_key) { + ereport(WARNING, + (errmsg("YAGPCC trying to submit already submitted query"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } + QueryKey::register_qkey(query_desc, nesting_level); + auto key = QueryKey::from_qdesc(query_desc); + auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); + if (!inserted) { + ereport(WARNING, (errmsg("YAGPCC duplicate query submit detected"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } } void EventSender::update_nested_counters(QueryDesc *query_desc) { if (!is_top_level_query(query_desc, nesting_level)) { - auto query_msg = get_query_message(query_desc); + auto &query = get_query(query_desc); nested_calls++; - double end_time = protots_to_double(query_msg->message->end_time()); - double start_time = protots_to_double(query_msg->message->start_time()); + double end_time = protots_to_double(query.message->end_time()); + double start_time = protots_to_double(query.message->start_time()); if (end_time >= start_time) { nested_timing += end_time - start_time; } else { @@ -391,6 +426,12 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { } } -EventSender::QueryItem::QueryItem(EventSender::QueryState st, - yagpcc::SetQueryReq *msg) - : state(st), message(msg) {} +bool EventSender::qdesc_submitted(QueryDesc *query_desc) { + if (query_desc->yagp_query_key == NULL) { + return false; + } + return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); +} + +EventSender::QueryItem::QueryItem(QueryState st) + : message(std::make_unique()), state(st) {} diff --git a/src/EventSender.h b/src/EventSender.h index f3dd1d2a528..4071d580ff9 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #define typeid __typeid extern "C" { @@ -11,12 +13,75 @@ extern "C" { } #undef typeid +#include "memory/gpdbwrappers.h" + class UDSConnector; struct QueryDesc; namespace yagpcc { class SetQueryReq; } +#include + +struct QueryKey { + int tmid; + int ssid; + int ccnt; + int nesting_level; + uintptr_t query_desc_addr; + + bool operator==(const QueryKey &other) const { + return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == + std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, + other.query_desc_addr); + } + + static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { + query_desc->yagp_query_key = + (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); + int32 tmid; + gpmon_gettmid(&tmid); + query_desc->yagp_query_key->tmid = tmid; + query_desc->yagp_query_key->ssid = gp_session_id; + query_desc->yagp_query_key->ccnt = gp_command_count; + query_desc->yagp_query_key->nesting_level = nesting_level; + query_desc->yagp_query_key->query_desc_addr = (uintptr_t)query_desc; + } + + static QueryKey from_qdesc(QueryDesc *query_desc) { + return { + .tmid = query_desc->yagp_query_key->tmid, + .ssid = query_desc->yagp_query_key->ssid, + .ccnt = query_desc->yagp_query_key->ccnt, + .nesting_level = query_desc->yagp_query_key->nesting_level, + .query_desc_addr = query_desc->yagp_query_key->query_desc_addr, + }; + } +}; + +// https://www.boost.org/doc/libs/1_35_0/doc/html/boost/hash_combine_id241013.html +template inline void hash_combine(std::size_t &seed, const T &v) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +namespace std { +template <> struct hash { + size_t operator()(const QueryKey &k) const noexcept { + size_t seed = hash{}(k.tmid); + hash_combine(seed, k.ssid); + hash_combine(seed, k.ccnt); + hash_combine(seed, k.nesting_level); + uintptr_t addr = k.query_desc_addr; + if constexpr (SIZE_MAX < UINTPTR_MAX) { + addr %= SIZE_MAX; + } + hash_combine(seed, addr); + return seed; + } +}; +} // namespace std + class EventSender { public: void executor_before_start(QueryDesc *query_desc, int eflags); @@ -31,30 +96,25 @@ class EventSender { ~EventSender(); private: - enum QueryState { UNKNOWN, SUBMIT, START, END, DONE }; + enum QueryState { SUBMIT, START, END, DONE }; struct QueryItem { - QueryState state = QueryState::UNKNOWN; - yagpcc::SetQueryReq *message = nullptr; + std::unique_ptr message; + QueryState state; - QueryItem(QueryState st, yagpcc::SetQueryReq *msg); - }; - - struct pair_hash { - std::size_t operator()(const std::pair &p) const { - auto h1 = std::hash{}(p.first); - auto h2 = std::hash{}(p.second); - return h1 ^ h2; - } + explicit QueryItem(QueryState st); }; - void update_query_state(QueryDesc *query_desc, QueryItem *query, - QueryState new_state, bool success = true); - QueryItem *get_query_message(QueryDesc *query_desc); + void update_query_state(QueryItem &query, QueryState new_state, + bool success = true); + QueryItem &get_query(QueryDesc *query_desc); + void submit_query(QueryDesc *query_desc); void collect_query_submit(QueryDesc *query_desc); + void report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status); void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); - void cleanup_messages(); void update_nested_counters(QueryDesc *query_desc); + bool qdesc_submitted(QueryDesc *query_desc); UDSConnector *connector = nullptr; int nesting_level = 0; @@ -63,5 +123,5 @@ class EventSender { #ifdef IC_TEARDOWN_HOOK ICStatistics ic_statistics; #endif - std::unordered_map, QueryItem, pair_hash> query_msgs; + std::unordered_map queries; }; \ No newline at end of file diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index f36cd030a39..929f0cf2681 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -60,14 +60,14 @@ std::string get_rg_name() { */ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - return (query_desc->gpmon_pkt && - query_desc->gpmon_pkt->u.qexec.key.tmid == 0) || - nesting_level == 0; + if (query_desc->yagp_query_key == NULL) { + return nesting_level == 0; + } + return query_desc->yagp_query_key->nesting_level == 0; } bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return (Gp_session_role == GP_ROLE_DISPATCH && - Config::report_nested_queries()) || + return need_report_nested_query() || is_top_level_query(query_desc, nesting_level); } diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 6dc39278bcd..4655433c806 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -58,21 +58,21 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_explain_state(query_desc, true); + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); - StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); *qi->mutable_template_plan_text() = char_to_trimmed_str( norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); - gpdb::pfree(es.str->data); - gpdb::pfree(norm_plan->data); + ya_gpdb::pfree(es.str->data); + ya_gpdb::pfree(norm_plan->data); } - gpdb::mem_ctx_switch_to(oldcxt); + ya_gpdb::mem_ctx_switch_to(oldcxt); } } @@ -82,7 +82,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { *qi->mutable_query_text() = char_to_trimmed_str( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); - char *norm_query = gpdb::gen_normquery(query_desc->sourceText); + char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = char_to_trimmed_str( norm_query, strlen(norm_query), Config::max_text_size()); } @@ -234,10 +234,10 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, return; } MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_analyze_state_json( + ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = ya_gpdb::get_analyze_state_json( query_desc, query_desc->instrument_options && Config::enable_analyze()); - gpdb::mem_ctx_switch_to(oldcxt); + ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { @@ -251,6 +251,6 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, auto trimmed_analyze = char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); - gpdb::pfree(es.str->data); + ya_gpdb::pfree(es.str->data); } } \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b5b70836db4..f8c4586126d 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -44,7 +44,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { auto data_size = req.ByteSize(); auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)gpdb::palloc(total_size); + uint8_t *buf = (uint8_t *)ya_gpdb::palloc(total_size); uint32_t *size_payload = (uint32_t *)buf; *size_payload = data_size; req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); @@ -67,7 +67,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, } else { YagpStat::report_send(total_size); } - gpdb::pfree(buf); + ya_gpdb::pfree(buf); } else { // log the error and go on log_tracing_failure(req, event); diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 25a85f086d1..d76b7c64e10 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -229,7 +229,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 1fba702a9f5..9d579a91a30 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -16,27 +16,104 @@ extern "C" { #include "stat_statements_parser/pg_stat_statements_ya_parser.h" } -void *gpdb::palloc(Size size) { return detail::wrap_throw(::palloc, size); } +namespace { -void *gpdb::palloc0(Size size) { return detail::wrap_throw(::palloc0, size); } +template +auto wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) { -char *gpdb::pstrdup(const char *str) { - return detail::wrap_throw(::pstrdup, str); + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult {}; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) { + result_holder.emplace(func(std::forward(args)...)); + } else { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) { + std::string err; + if (edata && edata->message) { + err = std::string(edata->message); + } else { + err = "Unknown error occurred"; + } + + if (edata) { + FreeErrorData(edata); + } + + if constexpr (Throws) { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) { + return RetType{}; + } else { + return; + } + } + + if constexpr (!std::is_void_v) { + return *std::move(result_holder); + } else { + return; + } +} + +template +auto wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) { + return wrap(std::forward(func), std::forward(args)...); } -char *gpdb::get_database_name(Oid dbid) noexcept { - return detail::wrap_noexcept(::get_database_name, dbid); +template +auto wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) { + return wrap(std::forward(func), std::forward(args)...); +} +} // namespace + +void *ya_gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } + +void *ya_gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } + +char *ya_gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } + +char *ya_gpdb::get_database_name(Oid dbid) noexcept { + return wrap_noexcept(::get_database_name, dbid); } -bool gpdb::split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept { - return detail::wrap_noexcept(SplitIdentifierString, rawstring, separator, - namelist); +bool ya_gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept { + return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState gpdb::get_explain_state(QueryDesc *query_desc, - bool costs) noexcept { - return detail::wrap_noexcept([&]() { +ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, + bool costs) noexcept { + return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); es.costs = costs; @@ -49,9 +126,9 @@ ExplainState gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept { - return detail::wrap_noexcept([&]() { +ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, + bool analyze) noexcept { + return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); es.analyze = analyze; @@ -70,79 +147,77 @@ ExplainState gpdb::get_analyze_state_json(QueryDesc *query_desc, }); } -Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options) { - return detail::wrap_throw(InstrAlloc, n, instrument_options); +Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options) { + return wrap_throw(InstrAlloc, n, instrument_options); } -HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull) { +HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, + bool *isnull) { if (!tupleDescriptor || !values || !isnull) throw std::runtime_error( "Invalid input parameters for heap tuple formation"); - return detail::wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); + return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void gpdb::pfree(void *pointer) noexcept { +void ya_gpdb::pfree(void *pointer) noexcept { // Note that ::pfree asserts that pointer != NULL. if (!pointer) return; - detail::wrap_noexcept(::pfree, pointer); + wrap_noexcept(::pfree, pointer); } -MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { +MemoryContext ya_gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { return MemoryContextSwitchTo(context); } -const char *gpdb::get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept { +const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept { if (!name) return nullptr; - return detail::wrap_noexcept(GetConfigOption, name, missing_ok, - restrict_superuser); + return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void gpdb::list_free(List *list) noexcept { +void ya_gpdb::list_free(List *list) noexcept { if (!list) return; - detail::wrap_noexcept(::list_free, list); + wrap_noexcept(::list_free, list); } CdbExplain_ShowStatCtx * -gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime) { +ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, + instr_time starttime) { if (!query_desc) throw std::runtime_error("Invalid query descriptor"); - return detail::wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, - starttime); + return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void gpdb::instr_end_loop(Instrumentation *instr) { +void ya_gpdb::instr_end_loop(Instrumentation *instr) { if (!instr) throw std::runtime_error("Invalid instrumentation pointer"); - detail::wrap_throw(::InstrEndLoop, instr); + wrap_throw(::InstrEndLoop, instr); } -char *gpdb::gen_normquery(const char *query) { - return detail::wrap_throw(::gen_normquery, query); +char *ya_gpdb::gen_normquery(const char *query) { + return wrap_throw(::gen_normquery, query); } -StringInfo gpdb::gen_normplan(const char *exec_plan) { +StringInfo ya_gpdb::gen_normplan(const char *exec_plan) { if (!exec_plan) throw std::runtime_error("Invalid execution plan string"); - return detail::wrap_throw(::gen_normplan, exec_plan); + return wrap_throw(::gen_normplan, exec_plan); } -char *gpdb::get_rg_name_for_id(Oid group_id) { - return detail::wrap_throw(GetResGroupNameForId, group_id); +char *ya_gpdb::get_rg_name_for_id(Oid group_id) { + return wrap_throw(GetResGroupNameForId, group_id); } -Oid gpdb::get_rg_id_by_session_id(int session_id) { - return detail::wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { + return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } \ No newline at end of file diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 437a5dd5d29..ad7ae96c362 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -16,88 +16,7 @@ extern "C" { #include #include -namespace gpdb { -namespace detail { - -template -auto wrap(Func &&func, Args &&...args) noexcept(!Throws) - -> decltype(func(std::forward(args)...)) { - - using RetType = decltype(func(std::forward(args)...)); - - // Empty struct for void return type. - struct VoidResult {}; - using ResultHolder = std::conditional_t, VoidResult, - std::optional>; - - bool success; - ErrorData *edata; - ResultHolder result_holder; - - PG_TRY(); - { - if constexpr (!std::is_void_v) { - result_holder.emplace(func(std::forward(args)...)); - } else { - func(std::forward(args)...); - } - edata = NULL; - success = true; - } - PG_CATCH(); - { - MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - MemoryContextSwitchTo(oldctx); - FlushErrorState(); - success = false; - } - PG_END_TRY(); - - if (!success) { - std::string err; - if (edata && edata->message) { - err = std::string(edata->message); - } else { - err = "Unknown error occurred"; - } - - if (edata) { - FreeErrorData(edata); - } - - if constexpr (Throws) { - throw std::runtime_error(err); - } - - if constexpr (!std::is_void_v) { - return RetType{}; - } else { - return; - } - } - - if constexpr (!std::is_void_v) { - return *std::move(result_holder); - } else { - return; - } -} - -template -auto wrap_throw(Func &&func, Args &&...args) - -> decltype(func(std::forward(args)...)) { - return detail::wrap(std::forward(func), - std::forward(args)...); -} - -template -auto wrap_noexcept(Func &&func, Args &&...args) noexcept - -> decltype(func(std::forward(args)...)) { - return detail::wrap(std::forward(func), - std::forward(args)...); -} -} // namespace detail +namespace ya_gpdb { // Functions that call palloc(). // Make sure correct memory context is set. @@ -128,4 +47,4 @@ const char *get_config_option(const char *name, bool missing_ok, void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace gpdb +} // namespace ya_gpdb From 027c0cdb1096400d7458ef7c5033754c8dfd83d3 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Thu, 4 Sep 2025 13:26:16 +0300 Subject: [PATCH 091/167] [yagp_hooks_collector] Add regression tests, ANALYZE text output, and UTF-8 trimming Add PG-style regression tests. Enable sending EXPLAIN ANALYZE as text. Add utility statement hook coverage. Implement UTF-8 safe trimming: discard partial multi-byte characters at cut boundaries. Clean up stray gmon.out. --- expected/yagp_cursors.out | 165 +++++++++++ expected/yagp_dist.out | 177 ++++++++++++ expected/yagp_select.out | 138 +++++++++ expected/yagp_utf8_trim.out | 66 +++++ expected/yagp_utility.out | 272 ++++++++++++++++++ metric.md | 49 ++-- sql/yagp_cursors.sql | 83 ++++++ sql/yagp_dist.sql | 86 ++++++ sql/yagp_select.sql | 67 +++++ sql/yagp_utf8_trim.sql | 43 +++ sql/yagp_utility.sql | 133 +++++++++ src/Config.cpp | 36 ++- src/Config.h | 5 + src/EventSender.cpp | 191 +++++++----- src/EventSender.h | 18 +- src/PgUtils.cpp | 5 - src/PgUtils.h | 3 - src/ProtoUtils.cpp | 69 +++-- src/ProtoUtils.h | 5 +- src/UDSConnector.cpp | 3 +- src/UDSConnector.h | 4 +- src/hook_wrappers.cpp | 60 +++- src/hook_wrappers.h | 3 + src/log/LogOps.cpp | 131 +++++++++ src/log/LogOps.h | 19 ++ src/log/LogSchema.cpp | 135 +++++++++ src/log/LogSchema.h | 166 +++++++++++ src/memory/gpdbwrappers.cpp | 13 +- src/memory/gpdbwrappers.h | 8 +- src/yagp_hooks_collector.c | 14 +- yagp_hooks_collector--1.0--1.1.sql | 113 ++++++++ ...--1.0.sql => yagp_hooks_collector--1.0.sql | 2 +- yagp_hooks_collector--1.1.sql | 95 ++++++ yagp_hooks_collector.control | 2 +- 34 files changed, 2224 insertions(+), 155 deletions(-) create mode 100644 expected/yagp_cursors.out create mode 100644 expected/yagp_dist.out create mode 100644 expected/yagp_select.out create mode 100644 expected/yagp_utf8_trim.out create mode 100644 expected/yagp_utility.out create mode 100644 sql/yagp_cursors.sql create mode 100644 sql/yagp_dist.sql create mode 100644 sql/yagp_select.sql create mode 100644 sql/yagp_utf8_trim.sql create mode 100644 sql/yagp_utility.sql create mode 100644 src/log/LogOps.cpp create mode 100644 src/log/LogOps.h create mode 100644 src/log/LogSchema.cpp create mode 100644 src/log/LogSchema.h create mode 100644 yagp_hooks_collector--1.0--1.1.sql rename sql/yagp_hooks_collector--1.0.sql => yagp_hooks_collector--1.0.sql (99%) create mode 100644 yagp_hooks_collector--1.1.sql diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out new file mode 100644 index 00000000000..9587c00b550 --- /dev/null +++ b/expected/yagp_cursors.out @@ -0,0 +1,165 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +-- DECLARE +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- DECLARE WITH HOLD +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(14 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- ROLLBACK +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(12 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- FETCH +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; + ?column? +---------- + 2 +(1 row) + +FETCH 1 IN cursor_stats_6; + ?column? +---------- + 3 +(1 row) + +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(18 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out new file mode 100644 index 00000000000..ebaf839601d --- /dev/null +++ b/expected/yagp_dist.out @@ -0,0 +1,177 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; +-- Hash distributed table +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; +SET yagpcc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; + id +---- + 1 +(1 row) + +RESET optimizer_enable_direct_dispatch; +RESET yagpcc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +SET yagpcc.logging_mode to 'TBL'; +-- Scan all segments. +select * from test_hash_dist; + id +---- + 1 +(1 row) + +DROP TABLE test_hash_dist; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------+--------------------- + -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist; | QUERY_STATUS_START + -1 | select * from test_hash_dist; | QUERY_STATUS_END + -1 | select * from test_hash_dist; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); + count +------- + 3 +(1 row) + +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------------------+--------------------- + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; + count +------- + 100 +(1 row) + +RESET yagpcc.logging_mode; +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + query_text | query_status +-----------------------------------------+--------------------- + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE +(12 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_select.out b/expected/yagp_select.out new file mode 100644 index 00000000000..4c4a0218150 --- /dev/null +++ b/expected/yagp_select.out @@ -0,0 +1,138 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; +-- Basic SELECT tests +SET yagpcc.logging_mode to 'TBL'; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +SELECT COUNT(*) FROM generate_series(1,10); + count +------- + 10 +(1 row) + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Transaction test +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +COMMIT; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- CTE test +SET yagpcc.logging_mode to 'TBL'; +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + column1 +--------- + 1 + 2 +(2 rows) + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-----------------------------+--------------------- + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE + | SELECT * FROM t; | +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Prepared statement test +SET yagpcc.logging_mode to 'TBL'; +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; + ?column? +---------- + 1 +(1 row) + +DEALLOCATE test_stmt; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------+--------------------- + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/expected/yagp_utf8_trim.out b/expected/yagp_utf8_trim.out new file mode 100644 index 00000000000..194ee6b3609 --- /dev/null +++ b/expected/yagp_utf8_trim.out @@ -0,0 +1,66 @@ +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM yagpcc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; +SET yagpcc.enable TO TRUE; +-- Test 1: 1 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; + ?column? +------------ + HelloWorld +(1 row) + +RESET yagpcc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 2: 2 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; + ?column? +------------- + РУССКИЙЯЗЫК +(1 row) + +RESET yagpcc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 3: 4 byte chars +SET yagpcc.max_text_size to 21; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; + ?column? +---------- + 😀 +(1 row) + +RESET yagpcc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET yagpcc.max_text_size; +RESET yagpcc.logging_mode; +RESET yagpcc.enable; +DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out new file mode 100644 index 00000000000..03c17713575 --- /dev/null +++ b/expected/yagp_utility.out @@ -0,0 +1,272 @@ +CREATE EXTENSION yagp_hooks_collector; +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE test_table (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE + -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE test_table; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Partitioning +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +NOTICE: CREATE TABLE will create partition "pt_test_1_prt_1" for table "pt_test" +NOTICE: CREATE TABLE will create partition "pt_test_1_prt_2" for table "pt_test" +DROP TABLE pt_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Views and Functions +SET yagpcc.logging_mode to 'TBL'; +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE + -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT + -1 | DROP VIEW test_view; | QUERY_STATUS_DONE + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(10 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Transaction Operations +SET yagpcc.logging_mode to 'TBL'; +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; +BEGIN; +SAVEPOINT sp2; +ABORT; +BEGIN; +ROLLBACK; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(18 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- DML Operations +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE dml_test (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE + -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(6 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- COPY Operations +SET yagpcc.logging_mode to 'TBL'; +CREATE TABLE copy_test (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +COPY (SELECT 1) TO STDOUT; +1 +DROP TABLE copy_test; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- Prepared Statements and error during execute +SET yagpcc.logging_mode to 'TBL'; +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +ERROR: division by zero +DEALLOCATE test_prep; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR + -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT + -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(8 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +-- GUC Settings +SET yagpcc.logging_mode to 'TBL'; +SET yagpcc.report_nested_queries TO FALSE; +RESET yagpcc.report_nested_queries; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT +(6 rows) + +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + t +--- + t +(1 row) + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/metric.md b/metric.md index 2d198391a67..5df56877edb 100644 --- a/metric.md +++ b/metric.md @@ -1,32 +1,33 @@ ## YAGP Hooks Collector Metrics -### States -A Postgres process goes through 4 executor functions to execute a query: -1) `ExecutorStart()` - resource allocation for the query. -2) `ExecutorRun()` - query execution. -3) `ExecutorFinish()` - cleanup. -4) `ExecutorEnd()` - cleanup. +### States +A Postgres process goes through 4 executor functions to execute a query: +1) `ExecutorStart()` - resource allocation for the query. +2) `ExecutorRun()` - query execution. +3) `ExecutorFinish()` - cleanup. +4) `ExecutorEnd()` - cleanup. -yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: ``` submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done ``` -### Key Points -- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. -- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. -- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. -- Nested queries on _Dispatcher_ become top-level on _Execute_. -- Each process (_Dispatcher_/_Execute_) sends its own statistics. +### Key Points +- Some queries may skip the _end_ state, then the _end_ statistics is sent during _done_. +- If a query finishes with an error (`METRICS_QUERY_ERROR`), or is cancelled (`METRICS_QUERY_CANCELLED`), statistics is sent at _done_. +- Some statistics is calculated as the difference between the current global metric and the previous. The initial snapshot is taken at submit, and at _end_/_done_ the diff is calculated. +- Nested queries on _Dispatcher_ become top-level on _Execute_. +- Each process (_Dispatcher_/_Execute_) sends its own statistics -### Notations -- **S** = Submit event. -- **T** = Start event. -- **E** = End event. -- **D** = Done event. -- **DIFF** = current_value - submit_value (submit event). -- **ABS** = Absolute value, or where diff is not applicable, the value taken. -- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. +### Notations +- **S** = Submit event. +- **T** = Start event. +- **E** = End event. +- **D** = Done event. +- **DIFF** = current_value - submit_value (submit event). +- **ABS** = Absolute value, or where diff is not applicable, the value taken. +- **Local*** - Statistics that starts counting from zero for each new query. A nested query is also considered new. +- **Node** - PG process, either a `Query Dispatcher` (on master) or an `Execute` (on segment). ### Statistics Table @@ -36,7 +37,7 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `runningTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | Wall clock time | | `userTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat utime | | `kernelTimeSeconds` | double | E, D | DIFF | - | Node | + | + | seconds | /proc/pid/stat stime | -| `vsize` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat vsize | +| `vsize` | uint64 | E, D | ABS | - | Node | + | + | bytes | /proc/pid/stat vsize | | `rss` | uint64 | E, D | ABS | - | Node | + | + | pages | /proc/pid/stat rss | | `VmSizeKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmSize | | `VmPeakKb` | uint64 | E, D | ABS | - | Node | + | + | KB | /proc/pid/status VmPeak | @@ -108,13 +109,13 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `userName` | string | All | ABS | - | Cluster | + | - | text | Session user | | `databaseName` | string | All | ABS | - | Cluster | + | - | text | Database name | | `rsgname` | string | All | ABS | - | Cluster | + | - | text | Resource group name | -| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE JSON | +| `analyze_text` | string | D | ABS | - | Cluster | + | - | text | EXPLAIN ANALYZE | | **AdditionalQueryInfo** | | | | | | | | | | | `nested_level` | int64 | All | ABS | - | Node | + | + | count | Current nesting level | | `error_message` | string | D | ABS | - | Node | + | + | text | Error message | | `slice_id` | int64 | All | ABS | - | Node | + | + | id | Slice ID | | **QueryKey** | | | | | | | | | | -| `tmid` | int32 | All | ABS | - | Node | + | + | id | Time ID | +| `tmid` | int32 | All | ABS | - | Node | + | + | id | Transaction start time | | `ssid` | int32 | All | ABS | - | Node | + | + | id | Session ID | | `ccnt` | int32 | All | ABS | - | Node | + | + | count | Command counter | | **SegmentKey** | | | | | | | | | | diff --git a/sql/yagp_cursors.sql b/sql/yagp_cursors.sql new file mode 100644 index 00000000000..5d5bde58110 --- /dev/null +++ b/sql/yagp_cursors.sql @@ -0,0 +1,83 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; + +-- DECLARE +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- DECLARE WITH HOLD +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- ROLLBACK +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- FETCH +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; +FETCH 1 IN cursor_stats_6; +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_dist.sql b/sql/yagp_dist.sql new file mode 100644 index 00000000000..b837ef05335 --- /dev/null +++ b/sql/yagp_dist.sql @@ -0,0 +1,86 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; + +-- Hash distributed table + +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; + +SET yagpcc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; +RESET optimizer_enable_direct_dispatch; + +RESET yagpcc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +SET yagpcc.logging_mode to 'TBL'; + +-- Scan all segments. +select * from test_hash_dist; + +DROP TABLE test_hash_dist; +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; + +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); + +SET yagpcc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; +RESET yagpcc.logging_mode; + +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_select.sql b/sql/yagp_select.sql new file mode 100644 index 00000000000..4038c6b7b63 --- /dev/null +++ b/sql/yagp_select.sql @@ -0,0 +1,67 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; +SET yagpcc.enable_utility TO FALSE; + +-- Basic SELECT tests +SET yagpcc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Transaction test +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +SELECT 1; +COMMIT; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- CTE test +SET yagpcc.logging_mode to 'TBL'; + +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Prepared statement test +SET yagpcc.logging_mode to 'TBL'; + +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; +DEALLOCATE test_stmt; + +RESET yagpcc.logging_mode; +SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/sql/yagp_utf8_trim.sql b/sql/yagp_utf8_trim.sql new file mode 100644 index 00000000000..c0fdcce24a5 --- /dev/null +++ b/sql/yagp_utf8_trim.sql @@ -0,0 +1,43 @@ +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM yagpcc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; + +SET yagpcc.enable TO TRUE; + +-- Test 1: 1 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; +RESET yagpcc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + +-- Test 2: 2 byte chars +SET yagpcc.max_text_size to 19; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; +RESET yagpcc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + +-- Test 3: 4 byte chars +SET yagpcc.max_text_size to 21; +SET yagpcc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; +RESET yagpcc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET yagpcc.max_text_size; +RESET yagpcc.logging_mode; +RESET yagpcc.enable; + +DROP EXTENSION yagp_hooks_collector; diff --git a/sql/yagp_utility.sql b/sql/yagp_utility.sql new file mode 100644 index 00000000000..b4cca6f5421 --- /dev/null +++ b/sql/yagp_utility.sql @@ -0,0 +1,133 @@ +CREATE EXTENSION yagp_hooks_collector; + +CREATE OR REPLACE FUNCTION yagp_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.report_nested_queries TO TRUE; + +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE test_table (a int, b text); +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Partitioning +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Views and Functions +SET yagpcc.logging_mode to 'TBL'; + +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Transaction Operations +SET yagpcc.logging_mode to 'TBL'; + +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; + +BEGIN; +SAVEPOINT sp2; +ABORT; + +BEGIN; +ROLLBACK; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- DML Operations +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE dml_test (a int, b text); +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- COPY Operations +SET yagpcc.logging_mode to 'TBL'; + +CREATE TABLE copy_test (a int); +COPY (SELECT 1) TO STDOUT; +DROP TABLE copy_test; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- Prepared Statements and error during execute +SET yagpcc.logging_mode to 'TBL'; + +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +DEALLOCATE test_prep; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +-- GUC Settings +SET yagpcc.logging_mode to 'TBL'; + +SET yagpcc.report_nested_queries TO FALSE; +RESET yagpcc.report_nested_queries; + +RESET yagpcc.logging_mode; + +SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT yagpcc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION yagp_status_order(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.report_nested_queries; +RESET yagpcc.enable_utility; diff --git a/src/Config.cpp b/src/Config.cpp index aef09fc7d73..dbd7e25b483 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,9 +16,16 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1024; // in KB -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = -1; // uninitialized state +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms +static int guc_logging_mode = LOG_MODE_UDS; +static bool guc_enable_utility = false; + +static const struct config_enum_entry logging_mode_options[] = { + {"uds", LOG_MODE_UDS, false /* hidden */}, + {"tbl", LOG_MODE_TBL, false}, + {NULL, 0, false}}; static std::unique_ptr> ignored_users_set = nullptr; @@ -92,9 +99,9 @@ void Config::init() { DefineCustomIntVariable( "yagpcc.max_text_size", - "Make yagpcc trim query texts longer than configured size", NULL, - &guc_max_text_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + "Make yagpcc trim query texts longer than configured size in bytes", NULL, + &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); DefineCustomIntVariable( "yagpcc.max_plan_size", @@ -106,18 +113,31 @@ void Config::init() { "yagpcc.min_analyze_time", "Sets the minimum execution time above which plans will be logged.", "Zero prints all plans. -1 turns this feature off.", - &guc_min_analyze_time, -1, -1, INT_MAX, PGC_USERSET, + &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); + + DefineCustomEnumVariable( + "yagpcc.logging_mode", "Logging mode: UDS or PG Table", NULL, + &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, + NULL); + + DefineCustomBoolVariable( + "yagpcc.enable_utility", "Collect utility statement stats", NULL, + &guc_enable_utility, false, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } std::string Config::uds_path() { return guc_uds_path; } bool Config::enable_analyze() { return guc_enable_analyze; } bool Config::enable_cdbstats() { return guc_enable_cdbstats; } bool Config::enable_collector() { return guc_enable_collector; } +bool Config::enable_utility() { return guc_enable_utility; } bool Config::report_nested_queries() { return guc_report_nested_queries; } -size_t Config::max_text_size() { return guc_max_text_size * 1024; } +size_t Config::max_text_size() { return guc_max_text_size; } size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } int Config::min_analyze_time() { return guc_min_analyze_time; }; +int Config::logging_mode() { return guc_logging_mode; } bool Config::filter_user(std::string username) { if (!ignored_users_set) { diff --git a/src/Config.h b/src/Config.h index eff83f0960a..7501c727a44 100644 --- a/src/Config.h +++ b/src/Config.h @@ -2,6 +2,9 @@ #include +#define LOG_MODE_UDS 0 +#define LOG_MODE_TBL 1 + class Config { public: static void init(); @@ -9,10 +12,12 @@ class Config { static bool enable_analyze(); static bool enable_cdbstats(); static bool enable_collector(); + static bool enable_utility(); static bool filter_user(std::string username); static bool report_nested_queries(); static size_t max_text_size(); static size_t max_plan_size(); static int min_analyze_time(); + static int logging_mode(); static void sync(); }; \ No newline at end of file diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 133d409b574..fee435a6dcc 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,6 +1,7 @@ #include "Config.h" #include "UDSConnector.h" #include "memory/gpdbwrappers.h" +#include "log/LogOps.h" #define typeid __typeid extern "C" { @@ -24,10 +25,82 @@ extern "C" { (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ Config::enable_analyze()) -void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { +static bool enable_utility = Config::enable_utility(); + +bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, + bool utility) { + if (!proto_verified) { + return false; + } if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { - return; + return false; + } + + switch (state) { + case QueryState::SUBMIT: + // Cache enable_utility at SUBMIT to ensure consistent behavior at DONE. + // Without caching, a query that sets enable_utility to false from true + // would be accepted at SUBMIT (guc is true) but rejected at DONE (guc + // is false), causing a leak. + enable_utility = Config::enable_utility(); + if (utility && enable_utility == false) { + return false; + } + // Sync config in case current query changes it. + Config::sync(); + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) { + QueryKey::register_qkey(query_desc, nesting_level); + return false; + } + if (is_top_level_query(query_desc, nesting_level)) { + nested_timing = 0; + nested_calls = 0; + } + break; + case QueryState::START: + if (!qdesc_submitted(query_desc)) { + collect_query_submit(query_desc, false /* utility */); + } + break; + case QueryState::DONE: + if (utility && enable_utility == false) { + return false; + } + default: + break; + } + + if (filter_query(query_desc)) { + return false; + } + if (!nesting_is_valid(query_desc, nesting_level)) { + return false; + } + + return true; +} + +bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, + const std::string &event, bool utility) { + bool clear_big_fields = false; + switch (Config::logging_mode()) { + case LOG_MODE_UDS: + clear_big_fields = UDSConnector::report_query(req, event); + break; + case LOG_MODE_TBL: + ya_gpdb::insert_log(req, utility); + clear_big_fields = false; + break; + default: + Assert(false); } + return clear_big_fields; +} + +void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata) { auto *query_desc = reinterpret_cast(arg); switch (status) { case METRICS_PLAN_NODE_INITIALIZE: @@ -36,7 +109,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { // TODO break; case METRICS_QUERY_SUBMIT: - collect_query_submit(query_desc); + collect_query_submit(query_desc, utility); break; case METRICS_QUERY_START: // no-op: executor_after_start is enough @@ -50,7 +123,7 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { case METRICS_QUERY_ERROR: case METRICS_QUERY_CANCELED: case METRICS_INNER_QUERY_DONE: - collect_query_done(query_desc, status); + collect_query_done(query_desc, utility, status, edata); break; default: ereport(FATAL, (errmsg("Unknown query status: %d", status))); @@ -58,18 +131,10 @@ void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg) { } void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { - if (!connector) { - return; - } - if (filter_query(query_desc)) { - return; - } - if (!qdesc_submitted(query_desc)) { - collect_query_submit(query_desc); - } - if (!need_collect(query_desc, nesting_level)) { + if (!verify_query(query_desc, QueryState::START, false /* utility*/)) { return; } + if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; @@ -88,16 +153,14 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { } void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!connector || !need_collect(query_desc, nesting_level)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if (!verify_query(query_desc, QueryState::START, false /* utility */)) { return; } + auto &query = get_query(query_desc); auto query_msg = query.message.get(); *query_msg->mutable_start_time() = current_ts(); - update_query_state(query, QueryState::START); + update_query_state(query, QueryState::START, false /* utility */); set_query_plan(query_msg, query_desc); if (need_collect_analyze()) { // Set up to track total elapsed time during query run. @@ -112,52 +175,37 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { } yagpcc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); - if (connector->report_query(*query_msg, "started")) { + if (log_query_req(*query_msg, "started", false /* utility */)) { clear_big_fields(query_msg); } std::swap(stats, *query_msg->mutable_query_metrics()); } void EventSender::executor_end(QueryDesc *query_desc) { - if (!connector || !need_collect(query_desc, nesting_level)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { + if (!verify_query(query_desc, QueryState::END, false /* utility */)) { return; } + auto &query = get_query(query_desc); auto *query_msg = query.message.get(); *query_msg->mutable_end_time() = current_ts(); - update_query_state(query, QueryState::END); + update_query_state(query, QueryState::END, false /* utility */); if (is_top_level_query(query_desc, nesting_level)) { set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, nested_timing); } else { set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); } - if (connector->report_query(*query_msg, "ended")) { + if (log_query_req(*query_msg, "ended", false /* utility */)) { clear_big_fields(query_msg); } } -void EventSender::collect_query_submit(QueryDesc *query_desc) { - if (!connector) { - return; - } - Config::sync(); - // Register qkey for a nested query we won't report, - // so we can detect nesting_level > 0 and skip reporting at end/done. - if (!need_report_nested_query() && nesting_level > 0) { - QueryKey::register_qkey(query_desc, nesting_level); - return; - } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; - } - if (!need_collect(query_desc, nesting_level)) { +void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { + if (!verify_query(query_desc, QueryState::SUBMIT, utility)) { return; } + submit_query(query_desc); auto &query = get_query(query_desc); auto *query_msg = query.message.get(); @@ -167,7 +215,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { set_qi_nesting_level(query_msg, nesting_level); set_qi_slice_id(query_msg); set_query_text(query_msg, query_desc); - if (connector->report_query(*query_msg, "submit")) { + if (log_query_req(*query_msg, "submit", utility)) { clear_big_fields(query_msg); } // take initial metrics snapshot so that we can safely take diff afterwards @@ -182,7 +230,8 @@ void EventSender::collect_query_submit(QueryDesc *query_desc) { } void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status) { + QueryMetricsStatus status, bool utility, + ErrorData *edata) { yagpcc::QueryStatus query_status; std::string msg; switch (status) { @@ -211,12 +260,20 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, (errmsg("Unexpected query status in query_done hook: %d", status))); } auto prev_state = query.state; - update_query_state(query, QueryState::DONE, + update_query_state(query, QueryState::DONE, utility, query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query.message.get(); query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { - set_qi_error_message(query_msg); + bool error_flushed = elog_message() == NULL; + if (error_flushed && edata->message == NULL) { + ereport(WARNING, (errmsg("YAGPCC missing error message"))); + ereport(DEBUG3, + (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + } else { + set_qi_error_message(query_msg, + error_flushed ? edata->message : elog_message()); + } } if (prev_state == START) { // We've missed ExecutorEnd call due to query cancel or error. It's @@ -230,12 +287,13 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), &ic_statistics); #endif - connector->report_query(*query_msg, msg); + (void)log_query_req(*query_msg, msg, utility); } -void EventSender::collect_query_done(QueryDesc *query_desc, - QueryMetricsStatus status) { - if (!connector || !need_collect(query_desc, nesting_level)) { +void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, + ErrorData *edata) { + if (!verify_query(query_desc, QueryState::DONE, utility)) { return; } @@ -258,10 +316,7 @@ void EventSender::collect_query_done(QueryDesc *query_desc, } auto &query = get_query(query_desc); - bool report = need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); - if (report) - report_query_done(query_desc, query, status); + report_query_done(query_desc, query, status, utility, edata); if (need_report_nested_query()) update_nested_counters(query_desc); @@ -276,7 +331,7 @@ void EventSender::ic_metrics_collect() { if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { return; } - if (!connector || gp_command_count == 0 || !Config::enable_collector() || + if (!proto_verified || gp_command_count == 0 || !Config::enable_collector() || Config::filter_user(get_user_name())) { return; } @@ -305,15 +360,12 @@ void EventSender::ic_metrics_collect() { } void EventSender::analyze_stats_collect(QueryDesc *query_desc) { - if (!connector || Gp_role != GP_ROLE_DISPATCH) { + if (!verify_query(query_desc, QueryState::END, false /* utility */)) { return; } - if (!need_collect(query_desc, nesting_level)) { + if (Gp_role != GP_ROLE_DISPATCH) { return; } - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg->mutable_end_time() = current_ts(); if (!query_desc->totaltime || !need_collect_analyze()) { return; } @@ -323,14 +375,17 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { double ms = query_desc->totaltime->total * 1000.0; if (ms >= Config::min_analyze_time()) { - set_analyze_plan_text_json(query_desc, query_msg); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + set_analyze_plan_text(query_desc, query_msg); } } EventSender::EventSender() { if (Config::enable_collector()) { try { - connector = new UDSConnector(); + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; } catch (const std::exception &e) { ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); } @@ -342,18 +397,16 @@ EventSender::EventSender() { EventSender::~EventSender() { for (const auto &[qkey, _] : queries) { - ereport(LOG, - (errmsg("YAGPCC query with missing done event: " - "tmid=%d ssid=%d ccnt=%d nlvl=%d", - qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); + ereport(LOG, (errmsg("YAGPCC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } - delete connector; } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP void EventSender::update_query_state(QueryItem &query, QueryState new_state, - bool success) { + bool utility, bool success) { switch (new_state) { case QueryState::SUBMIT: Assert(false); @@ -372,7 +425,7 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: - Assert(query.state == QueryState::END || !success); + Assert(query.state == QueryState::END || !success || utility); query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); break; default: diff --git a/src/EventSender.h b/src/EventSender.h index 4071d580ff9..4afdf1e14a4 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -87,7 +87,8 @@ class EventSender { void executor_before_start(QueryDesc *query_desc, int eflags); void executor_after_start(QueryDesc *query_desc, int eflags); void executor_end(QueryDesc *query_desc); - void query_metrics_collect(QueryMetricsStatus status, void *arg); + void query_metrics_collect(QueryMetricsStatus status, void *arg, bool utility, + ErrorData *edata = NULL); void ic_metrics_collect(); void analyze_stats_collect(QueryDesc *query_desc); void incr_depth() { nesting_level++; } @@ -105,18 +106,23 @@ class EventSender { explicit QueryItem(QueryState st); }; - void update_query_state(QueryItem &query, QueryState new_state, + static bool log_query_req(const yagpcc::SetQueryReq &req, + const std::string &event, bool utility); + bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); + void update_query_state(QueryItem &query, QueryState new_state, bool utility, bool success = true); QueryItem &get_query(QueryDesc *query_desc); void submit_query(QueryDesc *query_desc); - void collect_query_submit(QueryDesc *query_desc); + void collect_query_submit(QueryDesc *query_desc, bool utility); void report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status); - void collect_query_done(QueryDesc *query_desc, QueryMetricsStatus status); + QueryMetricsStatus status, bool utility, + ErrorData *edata = NULL); + void collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata = NULL); void update_nested_counters(QueryDesc *query_desc); bool qdesc_submitted(QueryDesc *query_desc); - UDSConnector *connector = nullptr; + bool proto_verified = false; int nesting_level = 0; int64_t nested_calls = 0; double nested_timing = 0; diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 929f0cf2681..fc58112bfaa 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -79,8 +79,3 @@ bool filter_query(QueryDesc *query_desc) { return gp_command_count == 0 || query_desc->sourceText == nullptr || !Config::enable_collector() || Config::filter_user(get_user_name()); } - -bool need_collect(QueryDesc *query_desc, int nesting_level) { - return !filter_query(query_desc) && - nesting_is_valid(query_desc, nesting_level); -} diff --git a/src/PgUtils.h b/src/PgUtils.h index ceb07c2e8e5..02f084c597a 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -12,6 +12,3 @@ bool is_top_level_query(QueryDesc *query_desc, int nesting_level); bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); bool need_report_nested_query(); bool filter_query(QueryDesc *query_desc); -bool need_collect(QueryDesc *query_desc, int nesting_level); -ExplainState get_explain_state(QueryDesc *query_desc, bool costs); -ExplainState get_analyze_state_json(QueryDesc *query_desc, bool analyze); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 4655433c806..f28714da6ec 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -24,6 +24,18 @@ extern "C" { #include #include +namespace { +constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); +constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); +constexpr uint8_t UTF8_MAX_SYMBOL_BYTES = 4; + +// Returns true if byte is the starting byte of utf8 +// character, false if byte is the continuation (10xxxxxx). +inline bool utf8_start_byte(uint8_t byte) { + return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; +} +} // namespace + google::protobuf::Timestamp current_ts() { google::protobuf::Timestamp current_ts; struct timeval tv; @@ -46,9 +58,26 @@ void set_segment_key(yagpcc::SegmentKey *key) { key->set_segindex(GpIdentity.segindex); } -inline std::string char_to_trimmed_str(const char *str, size_t len, - size_t lim) { - return std::string(str, std::min(len, lim)); +std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { + if (unlikely(str == nullptr)) { + return std::string(); + } + if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) { + return std::string(str, std::min(len, lim)); + } + + // Handle trimming of utf8 correctly, do not cut multi-byte characters. + size_t cut_pos = lim; + size_t visited_bytes = 1; + while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) { + if (utf8_start_byte(static_cast(str[cut_pos]))) { + break; + } + ++visited_bytes; + --cut_pos; + } + + return std::string(str, cut_pos); } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { @@ -61,10 +90,10 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { - *qi->mutable_plan_text() = char_to_trimmed_str(es.str->data, es.str->len, - Config::max_plan_size()); + *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, + Config::max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = char_to_trimmed_str( + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( norm_plan->data, norm_plan->len, Config::max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); @@ -79,11 +108,11 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = char_to_trimmed_str( + *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), Config::max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = char_to_trimmed_str( + *qi->mutable_template_query_text() = trim_str_shrink_utf8( norm_query, strlen(norm_query), Config::max_text_size()); } } @@ -103,7 +132,8 @@ void set_query_info(yagpcc::SetQueryReq *req) { if (Gp_session_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); - qi->set_databasename(get_db_name()); + if (IsTransactionState()) + qi->set_databasename(get_db_name()); qi->set_rsgname(get_rg_name()); } } @@ -118,11 +148,10 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req) { +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg) { auto aqi = req->mutable_add_info(); - auto error = elog_message(); *aqi->mutable_error_message() = - char_to_trimmed_str(error, strlen(error), Config::max_text_size()); + trim_str_shrink_utf8(err_msg, strlen(err_msg), Config::max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -226,8 +255,7 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text_json(QueryDesc *query_desc, - yagpcc::SetQueryReq *req) { +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. if (!IsTransactionState() || !query_desc->plannedstmt) { @@ -235,7 +263,7 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, } MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_analyze_state_json( + ExplainState es = ya_gpdb::get_analyze_state( query_desc, query_desc->instrument_options && Config::enable_analyze()); ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { @@ -243,14 +271,9 @@ void set_analyze_plan_text_json(QueryDesc *query_desc, if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { es.str->data[--es.str->len] = '\0'; } - // Convert JSON array to JSON object. - if (es.str->len > 0) { - es.str->data[0] = '{'; - es.str->data[es.str->len - 1] = '}'; - } - auto trimmed_analyze = - char_to_trimmed_str(es.str->data, es.str->len, Config::max_plan_size()); + auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, + Config::max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); ya_gpdb::pfree(es.str->data); } -} \ No newline at end of file +} diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 8287b3de7ea..725a634f765 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -12,12 +12,11 @@ void clear_big_fields(yagpcc::SetQueryReq *req); void set_query_info(yagpcc::SetQueryReq *req); void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req); +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text_json(QueryDesc *query_desc, - yagpcc::SetQueryReq *message); \ No newline at end of file +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message); \ No newline at end of file diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index f8c4586126d..b6af303218d 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -2,6 +2,7 @@ #include "Config.h" #include "YagpStat.h" #include "memory/gpdbwrappers.h" +#include "log/LogOps.h" #include #include @@ -16,8 +17,6 @@ extern "C" { #include "postgres.h" } -UDSConnector::UDSConnector() { GOOGLE_PROTOBUF_VERIFY_VERSION; } - static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { ereport(LOG, diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 67504fc8529..f0dfcb77a3f 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -4,6 +4,6 @@ class UDSConnector { public: - UDSConnector(); - bool report_query(const yagpcc::SetQueryReq &req, const std::string &event); + bool static report_query(const yagpcc::SetQueryReq &req, + const std::string &event); }; \ No newline at end of file diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index d76b7c64e10..07ac511d546 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -32,6 +32,7 @@ static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif +static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, @@ -44,6 +45,10 @@ static void ya_ic_teardown_hook(ChunkTransportState *transportStates, #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif +static void ya_process_utility_hook(Node *parsetree, const char *queryString, + ProcessUtilityContext context, + ParamListInfo params, DestReceiver *dest, + char *completionTag); static EventSender *sender = nullptr; @@ -85,6 +90,8 @@ void hooks_init() { analyze_stats_collect_hook = ya_analyze_stats_collect_hook; #endif stat_statements_parser_init(); + previous_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = ya_process_utility_hook; } void hooks_deinit() { @@ -104,6 +111,7 @@ void hooks_deinit() { delete sender; } YagpStat::deinit(); + ProcessUtility_hook = previous_ProcessUtility_hook; } void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { @@ -165,7 +173,8 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg); + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, + arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); if (previous_query_info_collect_hook) { (*previous_query_info_collect_hook)(status, arg); } @@ -189,6 +198,55 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif +static void ya_process_utility_hook(Node *parsetree, const char *queryString, + ProcessUtilityContext context, + ParamListInfo params, DestReceiver *dest, + char *completionTag) { + /* Project utility data on QueryDesc to use existing logic */ + QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); + query_desc->sourceText = queryString; + + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_SUBMIT, (void *)query_desc, true /* utility */, + (ErrorData *)NULL); + + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ProcessUtility_hook) { + (*previous_ProcessUtility_hook)(parsetree, queryString, context, params, + dest, completionTag); + } else { + standard_ProcessUtility(parsetree, queryString, context, params, dest, + completionTag); + } + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_DONE, + (void *)query_desc, true /* utility */, (ErrorData *)NULL); + + pfree(query_desc); + } + PG_CATCH(); + { + ErrorData *edata; + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(oldctx); + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_ERROR, + (void *)query_desc, true /* utility */, edata); + + pfree(query_desc); + ReThrowError(edata); + } + PG_END_TRY(); +} + static void check_stats_loaded() { if (!YagpStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index c158f42cf1d..cfabf39485e 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -9,6 +9,9 @@ extern void hooks_deinit(); extern void yagp_functions_reset(); extern Datum yagp_functions_get(FunctionCallInfo fcinfo); +extern void init_log(); +extern void truncate_log(); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp new file mode 100644 index 00000000000..0868dd9fc1c --- /dev/null +++ b/src/log/LogOps.cpp @@ -0,0 +1,131 @@ +#include "protos/yagpcc_set_service.pb.h" + +#include "LogOps.h" +#include "LogSchema.h" + +extern "C" { +#include "postgres.h" + +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/xact.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/namespace.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_type.h" +#include "cdb/cdbvars.h" +#include "commands/tablecmds.h" +#include "funcapi.h" +#include "fmgr.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" +} + +void init_log() { + Oid namespaceId; + Oid relationId; + ObjectAddress tableAddr; + ObjectAddress schemaAddr; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + + /* Create table */ + relationId = heap_create_with_catalog( + log_relname.data() /* relname */, namespaceId /* namespace */, + 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, + InvalidOid /* reloftypeid */, GetUserId() /* owner */, + DescribeTuple() /* rel tuple */, NIL, InvalidOid /* relam */, + RELKIND_RELATION, RELPERSISTENCE_PERMANENT, RELSTORAGE_HEAP, false, false, + true, 0, ONCOMMIT_NOOP, NULL /* GP Policy */, (Datum)0, + false /* use_user_acl */, true, true, false /* valid_opts */, + false /* is_part_child */, false /* is part parent */, NULL); + + /* Make the table visible */ + CommandCounterIncrement(); + + /* Record dependency of the table on the schema */ + if (OidIsValid(relationId) && OidIsValid(namespaceId)) { + ObjectAddressSet(tableAddr, RelationRelationId, relationId); + ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); + + /* Table can be dropped only via DROP EXTENSION */ + recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); + } else { + ereport(NOTICE, (errmsg("YAGPCC failed to create log table or schema"))); + } + + /* Make changes visible */ + CommandCounterIncrement(); +} + +void insert_log(const yagpcc::SetQueryReq &req, bool utility) { + Oid namespaceId; + Oid relationId; + Relation rel; + HeapTuple tuple; + + /* Return if xact is not valid (needed for catalog lookups). */ + if (!IsTransactionState()) { + return; + } + + /* Return if extension was not loaded */ + namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); + if (!OidIsValid(namespaceId)) { + return; + } + + /* Return if the table was not created yet */ + relationId = get_relname_relid(log_relname.data(), namespaceId); + if (!OidIsValid(relationId)) { + return; + } + + bool nulls[natts_yagp_log]; + Datum values[natts_yagp_log]; + + memset(nulls, true, sizeof(nulls)); + memset(values, 0, sizeof(values)); + + extract_query_req(req, "", values, nulls); + nulls[attnum_yagp_log_utility] = false; + values[attnum_yagp_log_utility] = BoolGetDatum(utility); + + rel = heap_open(relationId, RowExclusiveLock); + + /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls + * back or aborts */ + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + frozen_heap_insert(rel, tuple); + + heap_freetuple(tuple); + /* Keep lock on rel until end of xact */ + heap_close(rel, NoLock); + + /* Make changes visible */ + CommandCounterIncrement(); +} + +void truncate_log() { + Oid namespaceId; + Oid relationId; + Relation relation; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + relationId = get_relname_relid(log_relname.data(), namespaceId); + + relation = heap_open(relationId, AccessExclusiveLock); + + /* Truncate the main table */ + heap_truncate_one_rel(relation); + + /* Keep lock on rel until end of xact */ + heap_close(relation, NoLock); + + /* Make changes visible */ + CommandCounterIncrement(); +} \ No newline at end of file diff --git a/src/log/LogOps.h b/src/log/LogOps.h new file mode 100644 index 00000000000..bad03d09a8f --- /dev/null +++ b/src/log/LogOps.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +extern "C" { +#include "postgres.h" +#include "fmgr.h" +} + +extern "C" { +/* CREATE TABLE yagpcc.__log (...); */ +void init_log(); + +/* TRUNCATE yagpcc.__log */ +void truncate_log(); +} + +/* INSERT INTO yagpcc.__log VALUES (...) */ +void insert_log(const yagpcc::SetQueryReq &req, bool utility); diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp new file mode 100644 index 00000000000..335a3103cfd --- /dev/null +++ b/src/log/LogSchema.cpp @@ -0,0 +1,135 @@ +#include "google/protobuf/reflection.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/timestamp.pb.h" + +#include "LogSchema.h" + +const std::unordered_map &proto_name_to_col_idx() { + static const auto name_col_idx = [] { + std::unordered_map map; + map.reserve(log_tbl_desc.size()); + + for (size_t idx = 0; idx < natts_yagp_log; ++idx) { + map.emplace(log_tbl_desc[idx].proto_field_name, idx); + } + + return map; + }(); + return name_col_idx; +} + +TupleDesc DescribeTuple() { + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log, false); + + for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { + TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), + log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, + 0 /* attdim */); + } + + return tupdesc; +} + +Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts) { + TimestampTz pgtimestamp = + (TimestampTz)ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); + pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; + return TimestampTzGetDatum(pgtimestamp); +} + +Datum field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg) { + using namespace google::protobuf; + + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return Int32GetDatum(reflection->GetInt32(msg, field)); + case FieldDescriptor::CPPTYPE_INT64: + return Int64GetDatum(reflection->GetInt64(msg, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return Int64GetDatum(reflection->GetUInt32(msg, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return Int64GetDatum( + static_cast(reflection->GetUInt64(msg, field))); + case FieldDescriptor::CPPTYPE_DOUBLE: + return Float8GetDatum(reflection->GetDouble(msg, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return Float4GetDatum(reflection->GetFloat(msg, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return BoolGetDatum(reflection->GetBool(msg, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return CStringGetTextDatum(reflection->GetEnum(msg, field)->name().data()); + case FieldDescriptor::CPPTYPE_STRING: + return CStringGetTextDatum(reflection->GetString(msg, field).c_str()); + default: + return (Datum)0; + } +} + +void process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls) { + + auto proto_idx_map = proto_name_to_col_idx(); + auto it = proto_idx_map.find(field_name); + + if (it == proto_idx_map.end()) { + ereport(NOTICE, + (errmsg("YAGPCC protobuf field %s is not registered in log table", + field_name.c_str()))); + return; + } + + int idx = it->second; + + if (!reflection->HasField(msg, field)) { + nulls[idx] = true; + return; + } + + if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() == "google.protobuf.Timestamp") { + const auto &ts = static_cast( + reflection->GetMessage(msg, field)); + values[idx] = protots_to_timestamptz(ts); + } else { + values[idx] = field_to_datum(field, reflection, msg); + } + nulls[idx] = false; + + return; +} + +void extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls) { + using namespace google::protobuf; + + const Descriptor *descriptor = msg.GetDescriptor(); + const Reflection *reflection = msg.GetReflection(); + + for (int i = 0; i < descriptor->field_count(); ++i) { + const FieldDescriptor *field = descriptor->field(i); + + // For now, we do not log any repeated fields plus they need special + // treatment. + if (field->is_repeated()) { + continue; + } + + std::string curr_pref = prefix.empty() ? "" : prefix + "."; + std::string field_name = curr_pref + field->name().data(); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() != "google.protobuf.Timestamp") { + + if (reflection->HasField(msg, field)) { + const Message &nested = reflection->GetMessage(msg, field); + extract_query_req(nested, field_name, values, nulls); + } + } else { + process_field(field, reflection, msg, field_name, values, nulls); + } + } +} diff --git a/src/log/LogSchema.h b/src/log/LogSchema.h new file mode 100644 index 00000000000..f713c1e9b0e --- /dev/null +++ b/src/log/LogSchema.h @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include +#include + +extern "C" { +#include "postgres.h" +#include "access/htup_details.h" +#include "access/tupdesc.h" +#include "catalog/pg_type.h" +#include "utils/timestamp.h" +#include "utils/builtins.h" +} + +namespace google { +namespace protobuf { +class FieldDescriptor; +class Message; +class Reflection; +class Timestamp; +} // namespace protobuf +} // namespace google + +inline constexpr std::string_view schema_name = "yagpcc"; +inline constexpr std::string_view log_relname = "__log"; + +struct LogDesc { + std::string_view pg_att_name; + std::string_view proto_field_name; + Oid type_oid; +}; + +/* + * Definition of the log table structure. + * + * System stats collected as %lu (unsigned) may + * overflow INT8OID (signed), but this is acceptable. + */ +/* clang-format off */ +inline constexpr std::array log_tbl_desc = { + /* 8-byte aligned types first - Query Info */ + LogDesc{"query_id", "query_info.query_id", INT8OID}, + LogDesc{"plan_id", "query_info.plan_id", INT8OID}, + LogDesc{"nested_level", "add_info.nested_level", INT8OID}, + LogDesc{"slice_id", "add_info.slice_id", INT8OID}, + /* 8-byte aligned types - System Stats */ + LogDesc{"systemstat_vsize", "query_metrics.systemStat.vsize", INT8OID}, + LogDesc{"systemstat_rss", "query_metrics.systemStat.rss", INT8OID}, + LogDesc{"systemstat_vmsizekb", "query_metrics.systemStat.VmSizeKb", INT8OID}, + LogDesc{"systemstat_vmpeakkb", "query_metrics.systemStat.VmPeakKb", INT8OID}, + LogDesc{"systemstat_rchar", "query_metrics.systemStat.rchar", INT8OID}, + LogDesc{"systemstat_wchar", "query_metrics.systemStat.wchar", INT8OID}, + LogDesc{"systemstat_syscr", "query_metrics.systemStat.syscr", INT8OID}, + LogDesc{"systemstat_syscw", "query_metrics.systemStat.syscw", INT8OID}, + LogDesc{"systemstat_read_bytes", "query_metrics.systemStat.read_bytes", INT8OID}, + LogDesc{"systemstat_write_bytes", "query_metrics.systemStat.write_bytes", INT8OID}, + LogDesc{"systemstat_cancelled_write_bytes", "query_metrics.systemStat.cancelled_write_bytes", INT8OID}, + /* 8-byte aligned types - Metric Instrumentation */ + LogDesc{"instrumentation_ntuples", "query_metrics.instrumentation.ntuples", INT8OID}, + LogDesc{"instrumentation_nloops", "query_metrics.instrumentation.nloops", INT8OID}, + LogDesc{"instrumentation_tuplecount", "query_metrics.instrumentation.tuplecount", INT8OID}, + LogDesc{"instrumentation_shared_blks_hit", "query_metrics.instrumentation.shared_blks_hit", INT8OID}, + LogDesc{"instrumentation_shared_blks_read", "query_metrics.instrumentation.shared_blks_read", INT8OID}, + LogDesc{"instrumentation_shared_blks_dirtied", "query_metrics.instrumentation.shared_blks_dirtied", INT8OID}, + LogDesc{"instrumentation_shared_blks_written", "query_metrics.instrumentation.shared_blks_written", INT8OID}, + LogDesc{"instrumentation_local_blks_hit", "query_metrics.instrumentation.local_blks_hit", INT8OID}, + LogDesc{"instrumentation_local_blks_read", "query_metrics.instrumentation.local_blks_read", INT8OID}, + LogDesc{"instrumentation_local_blks_dirtied", "query_metrics.instrumentation.local_blks_dirtied", INT8OID}, + LogDesc{"instrumentation_local_blks_written", "query_metrics.instrumentation.local_blks_written", INT8OID}, + LogDesc{"instrumentation_temp_blks_read", "query_metrics.instrumentation.temp_blks_read", INT8OID}, + LogDesc{"instrumentation_temp_blks_written", "query_metrics.instrumentation.temp_blks_written", INT8OID}, + LogDesc{"instrumentation_inherited_calls", "query_metrics.instrumentation.inherited_calls", INT8OID}, + /* 8-byte aligned types - Network Stats */ + LogDesc{"instrumentation_sent_total_bytes", "query_metrics.instrumentation.sent.total_bytes", INT8OID}, + LogDesc{"instrumentation_sent_tuple_bytes", "query_metrics.instrumentation.sent.tuple_bytes", INT8OID}, + LogDesc{"instrumentation_sent_chunks", "query_metrics.instrumentation.sent.chunks", INT8OID}, + LogDesc{"instrumentation_received_total_bytes", "query_metrics.instrumentation.received.total_bytes", INT8OID}, + LogDesc{"instrumentation_received_tuple_bytes", "query_metrics.instrumentation.received.tuple_bytes", INT8OID}, + LogDesc{"instrumentation_received_chunks", "query_metrics.instrumentation.received.chunks", INT8OID}, + /* 8-byte aligned types - Interconnect Stats and spilled bytes */ + LogDesc{"interconnect_total_recv_queue_size", "query_metrics.instrumentation.interconnect.total_recv_queue_size", INT8OID}, + LogDesc{"interconnect_recv_queue_size_counting_time", "query_metrics.instrumentation.interconnect.recv_queue_size_counting_time", INT8OID}, + LogDesc{"interconnect_total_capacity", "query_metrics.instrumentation.interconnect.total_capacity", INT8OID}, + LogDesc{"interconnect_capacity_counting_time", "query_metrics.instrumentation.interconnect.capacity_counting_time", INT8OID}, + LogDesc{"interconnect_total_buffers", "query_metrics.instrumentation.interconnect.total_buffers", INT8OID}, + LogDesc{"interconnect_buffer_counting_time", "query_metrics.instrumentation.interconnect.buffer_counting_time", INT8OID}, + LogDesc{"interconnect_active_connections_num", "query_metrics.instrumentation.interconnect.active_connections_num", INT8OID}, + LogDesc{"interconnect_retransmits", "query_metrics.instrumentation.interconnect.retransmits", INT8OID}, + LogDesc{"interconnect_startup_cached_pkt_num", "query_metrics.instrumentation.interconnect.startup_cached_pkt_num", INT8OID}, + LogDesc{"interconnect_mismatch_num", "query_metrics.instrumentation.interconnect.mismatch_num", INT8OID}, + LogDesc{"interconnect_crc_errors", "query_metrics.instrumentation.interconnect.crc_errors", INT8OID}, + LogDesc{"interconnect_snd_pkt_num", "query_metrics.instrumentation.interconnect.snd_pkt_num", INT8OID}, + LogDesc{"interconnect_recv_pkt_num", "query_metrics.instrumentation.interconnect.recv_pkt_num", INT8OID}, + LogDesc{"interconnect_disordered_pkt_num", "query_metrics.instrumentation.interconnect.disordered_pkt_num", INT8OID}, + LogDesc{"interconnect_duplicated_pkt_num", "query_metrics.instrumentation.interconnect.duplicated_pkt_num", INT8OID}, + LogDesc{"interconnect_recv_ack_num", "query_metrics.instrumentation.interconnect.recv_ack_num", INT8OID}, + LogDesc{"interconnect_status_query_msg_num", "query_metrics.instrumentation.interconnect.status_query_msg_num", INT8OID}, + LogDesc{"spill_totalbytes", "query_metrics.spill.totalBytes", INT8OID}, + /* 8-byte aligned types - Float and Timestamp */ + LogDesc{"systemstat_runningtimeseconds", "query_metrics.systemStat.runningTimeSeconds", FLOAT8OID}, + LogDesc{"systemstat_usertimeseconds", "query_metrics.systemStat.userTimeSeconds", FLOAT8OID}, + LogDesc{"systemstat_kerneltimeseconds", "query_metrics.systemStat.kernelTimeSeconds", FLOAT8OID}, + LogDesc{"instrumentation_firsttuple", "query_metrics.instrumentation.firsttuple", FLOAT8OID}, + LogDesc{"instrumentation_startup", "query_metrics.instrumentation.startup", FLOAT8OID}, + LogDesc{"instrumentation_total", "query_metrics.instrumentation.total", FLOAT8OID}, + LogDesc{"instrumentation_blk_read_time", "query_metrics.instrumentation.blk_read_time", FLOAT8OID}, + LogDesc{"instrumentation_blk_write_time", "query_metrics.instrumentation.blk_write_time", FLOAT8OID}, + LogDesc{"instrumentation_startup_time", "query_metrics.instrumentation.startup_time", FLOAT8OID}, + LogDesc{"instrumentation_inherited_time", "query_metrics.instrumentation.inherited_time", FLOAT8OID}, + LogDesc{"datetime", "datetime", TIMESTAMPTZOID}, + LogDesc{"submit_time", "submit_time", TIMESTAMPTZOID}, + LogDesc{"start_time", "start_time", TIMESTAMPTZOID}, + LogDesc{"end_time", "end_time", TIMESTAMPTZOID}, + /* 4-byte aligned types - Query Key */ + LogDesc{"tmid", "query_key.tmid", INT4OID}, + LogDesc{"ssid", "query_key.ssid", INT4OID}, + LogDesc{"ccnt", "query_key.ccnt", INT4OID}, + /* 4-byte aligned types - Segment Key */ + LogDesc{"dbid", "segment_key.dbid", INT4OID}, + LogDesc{"segid", "segment_key.segindex", INT4OID}, + LogDesc{"spill_filecount", "query_metrics.spill.fileCount", INT4OID}, + /* Variable-length types - Query Info */ + LogDesc{"generator", "query_info.generator", TEXTOID}, + LogDesc{"query_text", "query_info.query_text", TEXTOID}, + LogDesc{"plan_text", "query_info.plan_text", TEXTOID}, + LogDesc{"template_query_text", "query_info.template_query_text", TEXTOID}, + LogDesc{"template_plan_text", "query_info.template_plan_text", TEXTOID}, + LogDesc{"user_name", "query_info.userName", TEXTOID}, + LogDesc{"database_name", "query_info.databaseName", TEXTOID}, + LogDesc{"rsgname", "query_info.rsgname", TEXTOID}, + LogDesc{"analyze_text", "query_info.analyze_text", TEXTOID}, + LogDesc{"error_message", "add_info.error_message", TEXTOID}, + LogDesc{"query_status", "query_status", TEXTOID}, + /* Extra field */ + LogDesc{"utility", "", BOOLOID}, +}; +/* clang-format on */ + +inline constexpr size_t natts_yagp_log = log_tbl_desc.size(); +inline constexpr size_t attnum_yagp_log_utility = natts_yagp_log - 1; + +const std::unordered_map &proto_name_to_col_idx(); + +TupleDesc DescribeTuple(); + +Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts); + +Datum field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg); + +/* Process a single proto field and store in values/nulls arrays */ +void process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls); + +/* + * Extracts values from msg into values/nulls arrays. Caller must + * pre-init nulls[] to true (this function does net set nulls + * to true for nested messages if parent message is missing). + */ +void extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls); diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 9d579a91a30..0824a3a6808 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -1,4 +1,5 @@ #include "gpdbwrappers.h" +#include "log/LogOps.h" extern "C" { #include "postgres.h" @@ -126,8 +127,8 @@ ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept { +ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, + bool analyze) noexcept { return wrap_noexcept([&]() { ExplainState es; ExplainInitState(&es); @@ -136,7 +137,7 @@ ExplainState ya_gpdb::get_analyze_state_json(QueryDesc *query_desc, es.buffers = es.analyze; es.timing = es.analyze; es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_JSON; + es.format = EXPLAIN_FORMAT_TEXT; ExplainBeginOutput(&es); if (analyze) { ExplainPrintPlan(&es, query_desc); @@ -220,4 +221,8 @@ char *ya_gpdb::get_rg_name_for_id(Oid group_id) { Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); -} \ No newline at end of file +} + +void ya_gpdb::insert_log(const yagpcc::SetQueryReq &req, bool utility) { + return wrap_throw(::insert_log, req, utility); +} diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index ad7ae96c362..8f5f146cc67 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -16,6 +16,10 @@ extern "C" { #include #include +namespace yagpcc { +class SetQueryReq; +} // namespace yagpcc + namespace ya_gpdb { // Functions that call palloc(). @@ -27,8 +31,7 @@ char *get_database_name(Oid dbid) noexcept; bool split_identifier_string(char *rawstring, char separator, List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; -ExplainState get_analyze_state_json(QueryDesc *query_desc, - bool analyze) noexcept; +ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; Instrumentation *instr_alloc(size_t n, int instrument_options); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull); @@ -38,6 +41,7 @@ void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query); StringInfo gen_normplan(const char *executionPlan); char *get_rg_name_for_id(Oid group_id); +void insert_log(const yagpcc::SetQueryReq &req, bool utility); // Palloc-free functions. void pfree(void *pointer) noexcept; diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 2a9e7328e6d..9db73638b24 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -10,6 +10,8 @@ void _PG_init(void); void _PG_fini(void); PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); PG_FUNCTION_INFO_V1(yagp_stat_messages); +PG_FUNCTION_INFO_V1(yagp_init_log); +PG_FUNCTION_INFO_V1(yagp_truncate_log); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { @@ -30,4 +32,14 @@ Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { Datum yagp_stat_messages(PG_FUNCTION_ARGS) { return yagp_functions_get(fcinfo); -} \ No newline at end of file +} + +Datum yagp_init_log(PG_FUNCTION_ARGS) { + init_log(); + PG_RETURN_VOID(); +} + +Datum yagp_truncate_log(PG_FUNCTION_ARGS) { + truncate_log(); + PG_RETURN_VOID(); +} diff --git a/yagp_hooks_collector--1.0--1.1.sql b/yagp_hooks_collector--1.0--1.1.sql new file mode 100644 index 00000000000..959d4f235d1 --- /dev/null +++ b/yagp_hooks_collector--1.0--1.1.sql @@ -0,0 +1,113 @@ +/* yagp_hooks_collector--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION yagp_hooks_collector UPDATE TO '1.1'" to load this file. \quit + +CREATE SCHEMA yagpcc; + +-- Unlink existing objects from extension. +ALTER EXTENSION yagp_hooks_collector DROP VIEW yagp_stat_messages; +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION yagp_stat_messages_reset(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_segments(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_master(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); +ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); + +-- Now drop the objects. +DROP VIEW yagp_stat_messages; +DROP FUNCTION yagp_stat_messages_reset(); +DROP FUNCTION __yagp_stat_messages_f_on_segments(); +DROP FUNCTION __yagp_stat_messages_f_on_master(); +DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); +DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); + +-- Recreate functions and view in new schema. +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.stat_messages_reset() +RETURNS void +AS +$$ + SELECT yagpcc.__stat_messages_reset_f_on_master(); + SELECT yagpcc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagpcc.stat_messages AS + SELECT C.* + FROM yagpcc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM yagpcc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +-- Create new objects. +CREATE FUNCTION yagpcc.__init_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__init_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside yagpcc schema. +SELECT yagpcc.__init_log_on_master(); +SELECT yagpcc.__init_log_on_segments(); + +CREATE VIEW yagpcc.log AS + SELECT * FROM yagpcc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('yagpcc.__log') -- segments + ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION yagpcc.__truncate_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__truncate_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.truncate_log() +RETURNS void AS $$ +BEGIN + PERFORM yagpcc.__truncate_log_on_master(); + PERFORM yagpcc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/sql/yagp_hooks_collector--1.0.sql b/yagp_hooks_collector--1.0.sql similarity index 99% rename from sql/yagp_hooks_collector--1.0.sql rename to yagp_hooks_collector--1.0.sql index 88bbe4e0dc7..7ab4e1b2fb7 100644 --- a/sql/yagp_hooks_collector--1.0.sql +++ b/yagp_hooks_collector--1.0.sql @@ -15,7 +15,7 @@ LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagp_stat_messages_reset() RETURNS void -AS +AS $$ SELECT __yagp_stat_messages_reset_f_on_master(); SELECT __yagp_stat_messages_reset_f_on_segments(); diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql new file mode 100644 index 00000000000..657720a88f2 --- /dev/null +++ b/yagp_hooks_collector--1.1.sql @@ -0,0 +1,95 @@ +/* yagp_hooks_collector--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit + +CREATE SCHEMA yagpcc; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.stat_messages_reset() +RETURNS void +AS +$$ + SELECT yagpcc.__stat_messages_reset_f_on_master(); + SELECT yagpcc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'yagp_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW yagpcc.stat_messages AS + SELECT C.* + FROM yagpcc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM yagpcc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION yagpcc.__init_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__init_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside yagpcc schema. +SELECT yagpcc.__init_log_on_master(); +SELECT yagpcc.__init_log_on_segments(); + +CREATE VIEW yagpcc.log AS + SELECT * FROM yagpcc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('yagpcc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION yagpcc.__truncate_log_on_master() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__truncate_log_on_segments() +RETURNS void +AS 'MODULE_PATHNAME', 'yagp_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION yagpcc.truncate_log() +RETURNS void AS $$ +BEGIN + PERFORM yagpcc.__truncate_log_on_master(); + PERFORM yagpcc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/yagp_hooks_collector.control b/yagp_hooks_collector.control index b5539dd6462..cb5906a1302 100644 --- a/yagp_hooks_collector.control +++ b/yagp_hooks_collector.control @@ -1,5 +1,5 @@ # yagp_hooks_collector extension comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' -default_version = '1.0' +default_version = '1.1' module_pathname = '$libdir/yagp_hooks_collector' superuser = true From 7622935e7dc20c65b8e4ee8d8c5f6c40116b8ed5 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 10:17:05 +0300 Subject: [PATCH 092/167] [yagp_hooks_collector] Port backend infrastructure and adapt for Cloudberry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port GpscQueryKey to QueryDesc and workfile spill counters to workfile_mgr.c from gpdb. Update for Cloudberry API changes: ExplainInitState→NewExplainState, gpmon_gettmid→gp_gettmid, Gp_session_role→Gp_role, signature changes in standard_ExecutorRun, standard_ProcessUtility, InstrAlloc, CreateTemplateTupleDesc. Change test functions to SRF. Remove redundant jumbling copies. --- expected/yagp_cursors.out | 12 +- expected/yagp_dist.out | 12 +- expected/yagp_select.out | 12 +- expected/yagp_utility.out | 52 +- src/EventSender.cpp | 2 +- src/EventSender.h | 4 +- src/PgUtils.cpp | 2 +- src/ProtoUtils.cpp | 12 +- src/UDSConnector.cpp | 10 +- src/backend/tcop/pquery.c | 3 + .../utils/workfile_manager/workfile_mgr.c | 24 + src/hook_wrappers.cpp | 34 +- src/include/executor/execdesc.h | 11 + src/include/utils/workfile_mgr.h | 4 + src/log/LogOps.cpp | 12 +- src/log/LogSchema.cpp | 2 +- src/memory/gpdbwrappers.cpp | 48 +- src/memory/gpdbwrappers.h | 2 +- .../pg_stat_statements_ya_parser.c | 760 +----------------- src/yagp_hooks_collector.c | 34 +- yagp_hooks_collector--1.0--1.1.sql | 16 +- yagp_hooks_collector--1.0.sql | 6 +- yagp_hooks_collector--1.1.sql | 16 +- 23 files changed, 217 insertions(+), 873 deletions(-) diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index 9587c00b550..d251ddd3e1c 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -40,8 +40,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- DECLARE WITH HOLD SET yagpcc.logging_mode to 'TBL'; @@ -74,8 +73,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- ROLLBACK SET yagpcc.logging_mode to 'TBL'; @@ -105,8 +103,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- FETCH SET yagpcc.logging_mode to 'TBL'; @@ -155,8 +152,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out index ebaf839601d..5fd5ea5fb3e 100644 --- a/expected/yagp_dist.out +++ b/expected/yagp_dist.out @@ -46,8 +46,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) SET yagpcc.logging_mode to 'TBL'; -- Scan all segments. @@ -83,8 +82,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Replicated table CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ @@ -128,8 +126,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Partially distributed table (2 numsegments) SET allow_system_table_mods = ON; @@ -167,8 +164,7 @@ SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_statu SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_select.out b/expected/yagp_select.out index 4c4a0218150..b6e18dc862f 100644 --- a/expected/yagp_select.out +++ b/expected/yagp_select.out @@ -46,8 +46,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Transaction test SET yagpcc.logging_mode to 'TBL'; @@ -72,8 +71,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- CTE test SET yagpcc.logging_mode to 'TBL'; @@ -102,8 +100,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Prepared statement test SET yagpcc.logging_mode to 'TBL'; @@ -128,8 +125,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 03c17713575..057f7d7a556 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -17,7 +17,7 @@ SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.logging_mode to 'TBL'; CREATE TABLE test_table (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE INDEX test_idx ON test_table(a); ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; @@ -41,8 +41,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Partitioning SET yagpcc.logging_mode to 'TBL'; @@ -50,34 +49,16 @@ CREATE TABLE pt_test (a int, b int) DISTRIBUTED BY (a) PARTITION BY RANGE (a) (START (0) END (100) EVERY (50)); -NOTICE: CREATE TABLE will create partition "pt_test_1_prt_1" for table "pt_test" -NOTICE: CREATE TABLE will create partition "pt_test_1_prt_2" for table "pt_test" DROP TABLE pt_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| @@ -85,13 +66,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT -(10 rows) +(6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Views and Functions SET yagpcc.logging_mode to 'TBL'; @@ -118,8 +98,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Transaction Operations SET yagpcc.logging_mode to 'TBL'; @@ -159,13 +138,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- DML Operations SET yagpcc.logging_mode to 'TBL'; CREATE TABLE dml_test (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. INSERT INTO dml_test VALUES (1, 'test'); UPDATE dml_test SET b = 'updated' WHERE a = 1; @@ -186,13 +164,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- COPY Operations SET yagpcc.logging_mode to 'TBL'; CREATE TABLE copy_test (a int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. COPY (SELECT 1) TO STDOUT; 1 @@ -214,8 +191,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- Prepared Statements and error during execute SET yagpcc.logging_mode to 'TBL'; @@ -240,8 +216,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) -- GUC Settings SET yagpcc.logging_mode to 'TBL'; @@ -262,8 +237,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util SELECT yagpcc.truncate_log() IS NOT NULL AS t; t --- - t -(1 row) +(0 rows) DROP FUNCTION yagp_status_order(text); DROP EXTENSION yagp_hooks_collector; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index fee435a6dcc..d638d275548 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -169,7 +169,7 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { if (query_desc->totaltime == NULL) { MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL); + query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL, false); ya_gpdb::mem_ctx_switch_to(oldcxt); } } diff --git a/src/EventSender.h b/src/EventSender.h index 4afdf1e14a4..6e195eeacdf 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -23,6 +23,8 @@ class SetQueryReq; #include +extern void gp_gettmid(int32 *); + struct QueryKey { int tmid; int ssid; @@ -40,7 +42,7 @@ struct QueryKey { query_desc->yagp_query_key = (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); int32 tmid; - gpmon_gettmid(&tmid); + gp_gettmid(&tmid); query_desc->yagp_query_key->tmid = tmid; query_desc->yagp_query_key->ssid = gp_session_id; query_desc->yagp_query_key->ccnt = gp_command_count; diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index fc58112bfaa..96f46429643 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -72,7 +72,7 @@ bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { } bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_session_role == GP_ROLE_DISPATCH; + return Config::report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; } bool filter_query(QueryDesc *query_desc) { diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index f28714da6ec..aa8632477f5 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -24,6 +24,8 @@ extern "C" { #include #include +extern void gp_gettmid(int32 *); + namespace { constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); @@ -49,7 +51,7 @@ void set_query_key(yagpcc::QueryKey *key) { key->set_ccnt(gp_command_count); key->set_ssid(gp_session_id); int32 tmid = 0; - gpmon_gettmid(&tmid); + gp_gettmid(&tmid); key->set_tmid(tmid); } @@ -81,7 +83,7 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { } void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { + if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER @@ -106,7 +108,7 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { - if (Gp_session_role == GP_ROLE_DISPATCH && query_desc->sourceText) { + if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), @@ -118,7 +120,7 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } void clear_big_fields(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { + if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); qi->clear_template_plan_text(); @@ -129,7 +131,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { } void set_query_info(yagpcc::SetQueryReq *req) { - if (Gp_session_role == GP_ROLE_DISPATCH) { + if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); if (IsTransactionState()) diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index b6af303218d..a7eaed539f7 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -19,10 +19,9 @@ extern "C" { static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, const std::string &event) { - ereport(LOG, - (errmsg("Query {%d-%d-%d} %s tracing failed with error %s", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str(), strerror(errno)))); + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str()))); } bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, @@ -77,8 +76,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, // That's a very important error that should never happen, so make it // visible to an end-user and admins. ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %s", - strerror(errno)))); + (errmsg("Unable to create non-blocking socket connection %m"))); success = false; YagpStat::report_error(); } diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 532690f1d51..7c1dbc480bc 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -127,6 +127,9 @@ CreateQueryDesc(PlannedStmt *plannedstmt, if (Gp_role != GP_ROLE_EXECUTE) increment_command_count(); + /* null this field until set by YAGP Hooks collector */ + qd->yagp_query_key = NULL; + return qd; } diff --git a/src/backend/utils/workfile_manager/workfile_mgr.c b/src/backend/utils/workfile_manager/workfile_mgr.c index e5b311cf9ba..21b4463e5f1 100644 --- a/src/backend/utils/workfile_manager/workfile_mgr.c +++ b/src/backend/utils/workfile_manager/workfile_mgr.c @@ -192,6 +192,9 @@ static void unpin_workset(workfile_set *work_set); static bool proc_exit_hook_registered = false; +static uint64 total_bytes_written = 0; +static uint64 total_files_created = 0; + Datum gp_workfile_mgr_cache_entries(PG_FUNCTION_ARGS); Datum gp_workfile_mgr_used_diskspace(PG_FUNCTION_ARGS); @@ -371,6 +374,7 @@ RegisterFileWithSet(File file, workfile_set *work_set) localCtl.entries[file].work_set = work_set; work_set->num_files++; work_set->perquery->num_files++; + total_files_created++; /* Enforce the limit on number of files */ if (gp_workfile_limit_files_per_query > 0 && @@ -447,6 +451,7 @@ UpdateWorkFileSize(File file, uint64 newsize) (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("workfile per segment size limit exceeded"))); } + total_bytes_written += diff; } /* @@ -986,3 +991,22 @@ workfile_is_active(workfile_set *workfile) { return workfile ? workfile->active : false; } + +uint64 +WorkfileTotalBytesWritten(void) +{ + return total_bytes_written; +} + +uint64 +WorkfileTotalFilesCreated(void) +{ + return total_files_created; +} + +void +WorkfileResetBackendStats(void) +{ + total_bytes_written = 0; + total_files_created = 0; +} diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 07ac511d546..56c1da9f4f6 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -36,7 +36,7 @@ static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - long count); + uint64 count, bool execute_once); static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); @@ -45,10 +45,12 @@ static void ya_ic_teardown_hook(ChunkTransportState *transportStates, #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void ya_process_utility_hook(Node *parsetree, const char *queryString, +static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, - ParamListInfo params, DestReceiver *dest, - char *completionTag); + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc); static EventSender *sender = nullptr; @@ -127,14 +129,14 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { } void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - long count) { + uint64 count, bool execute_once) { get_sender()->incr_depth(); PG_TRY(); { if (previous_ExecutorRun_hook) - previous_ExecutorRun_hook(query_desc, direction, count); + previous_ExecutorRun_hook(query_desc, direction, count, execute_once); else - standard_ExecutorRun(query_desc, direction, count); + standard_ExecutorRun(query_desc, direction, count, execute_once); get_sender()->decr_depth(); } PG_CATCH(); @@ -198,10 +200,12 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif -static void ya_process_utility_hook(Node *parsetree, const char *queryString, +static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, - ParamListInfo params, DestReceiver *dest, - char *completionTag) { + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) { /* Project utility data on QueryDesc to use existing logic */ QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); query_desc->sourceText = queryString; @@ -214,11 +218,11 @@ static void ya_process_utility_hook(Node *parsetree, const char *queryString, PG_TRY(); { if (previous_ProcessUtility_hook) { - (*previous_ProcessUtility_hook)(parsetree, queryString, context, params, - dest, completionTag); + (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); } else { - standard_ProcessUtility(parsetree, queryString, context, params, dest, - completionTag); + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, + queryEnv, dest, qc); } get_sender()->decr_depth(); @@ -264,7 +268,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { const int ATTNUM = 6; check_stats_loaded(); auto stats = YagpStat::get_stats(); - TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM, false); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, 0 /* attdim */); TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index e3ecf31b664..e469945a4c5 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -22,6 +22,14 @@ struct CdbExplain_ShowStatCtx; /* private, in "cdb/cdbexplain.c" */ +typedef struct YagpQueryKey +{ + int tmid; /* transaction time */ + int ssid; /* session id */ + int ccnt; /* command count */ + int nesting_level; + uintptr_t query_desc_addr; +} YagpQueryKey; /* * SerializedParams is used to serialize external query parameters @@ -330,6 +338,9 @@ typedef struct QueryDesc /* This is always set NULL by the core system, but plugins can change it */ struct Instrumentation *totaltime; /* total time spent in ExecutorRun */ + + /* YAGP Hooks collector */ + YagpQueryKey *yagp_query_key; } QueryDesc; /* in pquery.c */ diff --git a/src/include/utils/workfile_mgr.h b/src/include/utils/workfile_mgr.h index dfbd17bca57..48c83620610 100644 --- a/src/include/utils/workfile_mgr.h +++ b/src/include/utils/workfile_mgr.h @@ -74,4 +74,8 @@ extern workfile_set *workfile_mgr_cache_entries_get_copy(int* num_actives); extern uint64 WorkfileSegspace_GetSize(void); extern bool workfile_is_active(workfile_set *workfile); +extern uint64 WorkfileTotalBytesWritten(void); +extern uint64 WorkfileTotalFilesCreated(void); +extern void WorkfileResetBackendStats(void); + #endif /* __WORKFILE_MGR_H__ */ diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index 0868dd9fc1c..cec9e33693a 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -37,12 +37,12 @@ void init_log() { relationId = heap_create_with_catalog( log_relname.data() /* relname */, namespaceId /* namespace */, 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, - InvalidOid /* reloftypeid */, GetUserId() /* owner */, - DescribeTuple() /* rel tuple */, NIL, InvalidOid /* relam */, - RELKIND_RELATION, RELPERSISTENCE_PERMANENT, RELSTORAGE_HEAP, false, false, - true, 0, ONCOMMIT_NOOP, NULL /* GP Policy */, (Datum)0, - false /* use_user_acl */, true, true, false /* valid_opts */, - false /* is_part_child */, false /* is part parent */, NULL); + InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, + DescribeTuple() /* rel tuple */, NIL, RELKIND_RELATION, + RELPERSISTENCE_PERMANENT, false, false, ONCOMMIT_NOOP, + NULL /* GP Policy */, (Datum)0, false /* use_user_acl */, true, true, + InvalidOid /* relrewrite */, NULL /* typaddress */, + false /* valid_opts */); /* Make the table visible */ CommandCounterIncrement(); diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp index 335a3103cfd..2fadcc46599 100644 --- a/src/log/LogSchema.cpp +++ b/src/log/LogSchema.cpp @@ -19,7 +19,7 @@ const std::unordered_map &proto_name_to_col_idx() { } TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log, false); + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log); for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 0824a3a6808..763e32e539c 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -7,6 +7,7 @@ extern "C" { #include "commands/dbcommands.h" #include "commands/resgroupcmds.h" #include "utils/builtins.h" +#include "utils/varlena.h" #include "nodes/pg_list.h" #include "commands/explain.h" #include "executor/instrument.h" @@ -115,41 +116,40 @@ bool ya_gpdb::split_identifier_string(char *rawstring, char separator, ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept { return wrap_noexcept([&]() { - ExplainState es; - ExplainInitState(&es); - es.costs = costs; - es.verbose = true; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); - ExplainPrintPlan(&es, query_desc); - ExplainEndOutput(&es); - return es; + ExplainState *es = NewExplainState(); + es->costs = costs; + es->verbose = true; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + ExplainPrintPlan(es, query_desc); + ExplainEndOutput(es); + return *es; }); } ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept { return wrap_noexcept([&]() { - ExplainState es; - ExplainInitState(&es); - es.analyze = analyze; - es.verbose = true; - es.buffers = es.analyze; - es.timing = es.analyze; - es.summary = es.analyze; - es.format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(&es); + ExplainState *es = NewExplainState(); + es->analyze = analyze; + es->verbose = true; + es->buffers = es->analyze; + es->timing = es->analyze; + es->summary = es->analyze; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); if (analyze) { - ExplainPrintPlan(&es, query_desc); - ExplainPrintExecStatsEnd(&es, query_desc); + ExplainPrintPlan(es, query_desc); + ExplainPrintExecStatsEnd(es, query_desc); } - ExplainEndOutput(&es); - return es; + ExplainEndOutput(es); + return *es; }); } -Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options) { - return wrap_throw(InstrAlloc, n, instrument_options); +Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options, + bool async_mode) { + return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 8f5f146cc67..920fc1ae6e7 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -32,7 +32,7 @@ bool split_identifier_string(char *rawstring, char separator, List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; -Instrumentation *instr_alloc(size_t n, int instrument_options); +Instrumentation *instr_alloc(size_t n, int instrument_options, bool async_mode); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull); CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 1c58d936093..c19805ce506 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -6,689 +6,48 @@ #include #include -#include "access/hash.h" -#include "executor/instrument.h" -#include "executor/execdesc.h" -#include "funcapi.h" +#include "common/hashfn.h" +#include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "miscadmin.h" -#include "parser/analyze.h" -#include "parser/parsetree.h" #include "parser/scanner.h" -#include "parser/gram.h" -#include "pgstat.h" -#include "storage/fd.h" -#include "storage/ipc.h" -#include "storage/spin.h" -#include "tcop/utility.h" #include "utils/builtins.h" #include "utils/memutils.h" +#include "utils/queryjumble.h" #include "pg_stat_statements_ya_parser.h" -static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; - -#define JUMBLE_SIZE 1024 /* query serialization buffer size */ - -/* - * Struct for tracking locations/lengths of constants during normalization - */ -typedef struct pgssLocationLen -{ - int location; /* start offset in query text */ - int length; /* length in bytes, or -1 to ignore */ -} pgssLocationLen; - -/* - * Working state for computing a query jumble and producing a normalized - * query string - */ -typedef struct pgssJumbleState -{ - /* Jumble of current query tree */ - unsigned char *jumble; - - /* Number of bytes used in jumble[] */ - Size jumble_len; - - /* Array of locations of constants that should be removed */ - pgssLocationLen *clocations; - - /* Allocated length of clocations array */ - int clocations_buf_size; - - /* Current number of valid entries in clocations array */ - int clocations_count; - - /* highest Param id we've seen, in order to start normalization correctly */ - int highest_extern_param_id; -} pgssJumbleState; +#ifndef ICONST +#define ICONST 276 +#endif +#ifndef FCONST +#define FCONST 277 +#endif +#ifndef SCONST +#define SCONST 278 +#endif +#ifndef BCONST +#define BCONST 279 +#endif +#ifndef XCONST +#define XCONST 280 +#endif -static void AppendJumble(pgssJumbleState *jstate, - const unsigned char *item, Size size); -static void JumbleQuery(pgssJumbleState *jstate, Query *query); -static void JumbleRangeTable(pgssJumbleState *jstate, List *rtable); -static void JumbleExpr(pgssJumbleState *jstate, Node *node); -static void RecordConstLocation(pgssJumbleState *jstate, int location); -static void fill_in_constant_lengths(pgssJumbleState *jstate, const char *query); +static void fill_in_constant_lengths(JumbleState *jstate, const char *query); static int comp_location(const void *a, const void *b); StringInfo gen_normplan(const char *execution_plan); static bool need_replace(int token); -void pgss_post_parse_analyze(ParseState *pstate, Query *query); -static char *generate_normalized_query(pgssJumbleState *jstate, const char *query, +static char *generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding); - void stat_statements_parser_init() -{ - prev_post_parse_analyze_hook = post_parse_analyze_hook; - post_parse_analyze_hook = pgss_post_parse_analyze; -} - -void stat_statements_parser_deinit() +void stat_statements_parser_init(void) { - post_parse_analyze_hook = prev_post_parse_analyze_hook; -} - -/* - * AppendJumble: Append a value that is substantive in a given query to - * the current jumble. - */ -static void -AppendJumble(pgssJumbleState *jstate, const unsigned char *item, Size size) -{ - unsigned char *jumble = jstate->jumble; - Size jumble_len = jstate->jumble_len; - - /* - * Whenever the jumble buffer is full, we hash the current contents and - * reset the buffer to contain just that hash value, thus relying on the - * hash to summarize everything so far. - */ - while (size > 0) - { - Size part_size; - - if (jumble_len >= JUMBLE_SIZE) - { - uint32 start_hash = hash_any(jumble, JUMBLE_SIZE); - - memcpy(jumble, &start_hash, sizeof(start_hash)); - jumble_len = sizeof(start_hash); - } - part_size = Min(size, JUMBLE_SIZE - jumble_len); - memcpy(jumble + jumble_len, item, part_size); - jumble_len += part_size; - item += part_size; - size -= part_size; - } - jstate->jumble_len = jumble_len; + EnableQueryId(); } -/* - * Wrappers around AppendJumble to encapsulate details of serialization - * of individual local variable elements. - */ -#define APP_JUMB(item) \ - AppendJumble(jstate, (const unsigned char *)&(item), sizeof(item)) -#define APP_JUMB_STRING(str) \ - AppendJumble(jstate, (const unsigned char *)(str), strlen(str) + 1) - -/* - * JumbleQuery: Selectively serialize the query tree, appending significant - * data to the "query jumble" while ignoring nonsignificant data. - * - * Rule of thumb for what to include is that we should ignore anything not - * semantically significant (such as alias names) as well as anything that can - * be deduced from child nodes (else we'd just be double-hashing that piece - * of information). - */ -void JumbleQuery(pgssJumbleState *jstate, Query *query) +void stat_statements_parser_deinit(void) { - Assert(IsA(query, Query)); - Assert(query->utilityStmt == NULL); - - APP_JUMB(query->commandType); - /* resultRelation is usually predictable from commandType */ - JumbleExpr(jstate, (Node *)query->cteList); - JumbleRangeTable(jstate, query->rtable); - JumbleExpr(jstate, (Node *)query->jointree); - JumbleExpr(jstate, (Node *)query->targetList); - JumbleExpr(jstate, (Node *)query->returningList); - JumbleExpr(jstate, (Node *)query->groupClause); - JumbleExpr(jstate, query->havingQual); - JumbleExpr(jstate, (Node *)query->windowClause); - JumbleExpr(jstate, (Node *)query->distinctClause); - JumbleExpr(jstate, (Node *)query->sortClause); - JumbleExpr(jstate, query->limitOffset); - JumbleExpr(jstate, query->limitCount); - /* we ignore rowMarks */ - JumbleExpr(jstate, query->setOperations); -} - -/* - * Jumble a range table - */ -static void -JumbleRangeTable(pgssJumbleState *jstate, List *rtable) -{ - ListCell *lc; - - foreach (lc, rtable) - { - RangeTblEntry *rte = (RangeTblEntry *)lfirst(lc); - - Assert(IsA(rte, RangeTblEntry)); - APP_JUMB(rte->rtekind); - switch (rte->rtekind) - { - case RTE_RELATION: - APP_JUMB(rte->relid); - break; - case RTE_SUBQUERY: - JumbleQuery(jstate, rte->subquery); - break; - case RTE_JOIN: - APP_JUMB(rte->jointype); - break; - case RTE_FUNCTION: - JumbleExpr(jstate, (Node *)rte->functions); - break; - case RTE_VALUES: - JumbleExpr(jstate, (Node *)rte->values_lists); - break; - case RTE_CTE: - - /* - * Depending on the CTE name here isn't ideal, but it's the - * only info we have to identify the referenced WITH item. - */ - APP_JUMB_STRING(rte->ctename); - APP_JUMB(rte->ctelevelsup); - break; - /* GPDB RTEs */ - case RTE_VOID: - break; - case RTE_TABLEFUNCTION: - JumbleQuery(jstate, rte->subquery); - JumbleExpr(jstate, (Node *)rte->functions); - break; - default: - ereport(ERROR, (errmsg("unrecognized RTE kind: %d", (int)rte->rtekind))); - break; - } - } -} - -/* - * Jumble an expression tree - * - * In general this function should handle all the same node types that - * expression_tree_walker() does, and therefore it's coded to be as parallel - * to that function as possible. However, since we are only invoked on - * queries immediately post-parse-analysis, we need not handle node types - * that only appear in planning. - * - * Note: the reason we don't simply use expression_tree_walker() is that the - * point of that function is to support tree walkers that don't care about - * most tree node types, but here we care about all types. We should complain - * about any unrecognized node type. - */ -static void -JumbleExpr(pgssJumbleState *jstate, Node *node) -{ - ListCell *temp; - - if (node == NULL) - return; - - /* Guard against stack overflow due to overly complex expressions */ - check_stack_depth(); - - /* - * We always emit the node's NodeTag, then any additional fields that are - * considered significant, and then we recurse to any child nodes. - */ - APP_JUMB(node->type); - - switch (nodeTag(node)) - { - case T_Var: - { - Var *var = (Var *)node; - - APP_JUMB(var->varno); - APP_JUMB(var->varattno); - APP_JUMB(var->varlevelsup); - } - break; - case T_Const: - { - Const *c = (Const *)node; - - /* We jumble only the constant's type, not its value */ - APP_JUMB(c->consttype); - /* Also, record its parse location for query normalization */ - RecordConstLocation(jstate, c->location); - } - break; - case T_Param: - { - Param *p = (Param *)node; - - APP_JUMB(p->paramkind); - APP_JUMB(p->paramid); - APP_JUMB(p->paramtype); - } - break; - case T_Aggref: - { - Aggref *expr = (Aggref *)node; - - APP_JUMB(expr->aggfnoid); - JumbleExpr(jstate, (Node *)expr->aggdirectargs); - JumbleExpr(jstate, (Node *)expr->args); - JumbleExpr(jstate, (Node *)expr->aggorder); - JumbleExpr(jstate, (Node *)expr->aggdistinct); - JumbleExpr(jstate, (Node *)expr->aggfilter); - } - break; - case T_WindowFunc: - { - WindowFunc *expr = (WindowFunc *)node; - - APP_JUMB(expr->winfnoid); - APP_JUMB(expr->winref); - JumbleExpr(jstate, (Node *)expr->args); - JumbleExpr(jstate, (Node *)expr->aggfilter); - } - break; - case T_ArrayRef: - { - ArrayRef *aref = (ArrayRef *)node; - - JumbleExpr(jstate, (Node *)aref->refupperindexpr); - JumbleExpr(jstate, (Node *)aref->reflowerindexpr); - JumbleExpr(jstate, (Node *)aref->refexpr); - JumbleExpr(jstate, (Node *)aref->refassgnexpr); - } - break; - case T_FuncExpr: - { - FuncExpr *expr = (FuncExpr *)node; - - APP_JUMB(expr->funcid); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_NamedArgExpr: - { - NamedArgExpr *nae = (NamedArgExpr *)node; - - APP_JUMB(nae->argnumber); - JumbleExpr(jstate, (Node *)nae->arg); - } - break; - case T_OpExpr: - case T_DistinctExpr: /* struct-equivalent to OpExpr */ - case T_NullIfExpr: /* struct-equivalent to OpExpr */ - { - OpExpr *expr = (OpExpr *)node; - - APP_JUMB(expr->opno); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_ScalarArrayOpExpr: - { - ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *)node; - - APP_JUMB(expr->opno); - APP_JUMB(expr->useOr); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_BoolExpr: - { - BoolExpr *expr = (BoolExpr *)node; - - APP_JUMB(expr->boolop); - JumbleExpr(jstate, (Node *)expr->args); - } - break; - case T_SubLink: - { - SubLink *sublink = (SubLink *)node; - - APP_JUMB(sublink->subLinkType); - JumbleExpr(jstate, (Node *)sublink->testexpr); - JumbleQuery(jstate, (Query *)sublink->subselect); - } - break; - case T_FieldSelect: - { - FieldSelect *fs = (FieldSelect *)node; - - APP_JUMB(fs->fieldnum); - JumbleExpr(jstate, (Node *)fs->arg); - } - break; - case T_FieldStore: - { - FieldStore *fstore = (FieldStore *)node; - - JumbleExpr(jstate, (Node *)fstore->arg); - JumbleExpr(jstate, (Node *)fstore->newvals); - } - break; - case T_RelabelType: - { - RelabelType *rt = (RelabelType *)node; - - APP_JUMB(rt->resulttype); - JumbleExpr(jstate, (Node *)rt->arg); - } - break; - case T_CoerceViaIO: - { - CoerceViaIO *cio = (CoerceViaIO *)node; - - APP_JUMB(cio->resulttype); - JumbleExpr(jstate, (Node *)cio->arg); - } - break; - case T_ArrayCoerceExpr: - { - ArrayCoerceExpr *acexpr = (ArrayCoerceExpr *)node; - - APP_JUMB(acexpr->resulttype); - JumbleExpr(jstate, (Node *)acexpr->arg); - } - break; - case T_ConvertRowtypeExpr: - { - ConvertRowtypeExpr *crexpr = (ConvertRowtypeExpr *)node; - - APP_JUMB(crexpr->resulttype); - JumbleExpr(jstate, (Node *)crexpr->arg); - } - break; - case T_CollateExpr: - { - CollateExpr *ce = (CollateExpr *)node; - - APP_JUMB(ce->collOid); - JumbleExpr(jstate, (Node *)ce->arg); - } - break; - case T_CaseExpr: - { - CaseExpr *caseexpr = (CaseExpr *)node; - - JumbleExpr(jstate, (Node *)caseexpr->arg); - foreach (temp, caseexpr->args) - { - CaseWhen *when = (CaseWhen *)lfirst(temp); - - Assert(IsA(when, CaseWhen)); - JumbleExpr(jstate, (Node *)when->expr); - JumbleExpr(jstate, (Node *)when->result); - } - JumbleExpr(jstate, (Node *)caseexpr->defresult); - } - break; - case T_CaseTestExpr: - { - CaseTestExpr *ct = (CaseTestExpr *)node; - - APP_JUMB(ct->typeId); - } - break; - case T_ArrayExpr: - JumbleExpr(jstate, (Node *)((ArrayExpr *)node)->elements); - break; - case T_RowExpr: - JumbleExpr(jstate, (Node *)((RowExpr *)node)->args); - break; - case T_RowCompareExpr: - { - RowCompareExpr *rcexpr = (RowCompareExpr *)node; - - APP_JUMB(rcexpr->rctype); - JumbleExpr(jstate, (Node *)rcexpr->largs); - JumbleExpr(jstate, (Node *)rcexpr->rargs); - } - break; - case T_CoalesceExpr: - JumbleExpr(jstate, (Node *)((CoalesceExpr *)node)->args); - break; - case T_MinMaxExpr: - { - MinMaxExpr *mmexpr = (MinMaxExpr *)node; - - APP_JUMB(mmexpr->op); - JumbleExpr(jstate, (Node *)mmexpr->args); - } - break; - case T_XmlExpr: - { - XmlExpr *xexpr = (XmlExpr *)node; - - APP_JUMB(xexpr->op); - JumbleExpr(jstate, (Node *)xexpr->named_args); - JumbleExpr(jstate, (Node *)xexpr->args); - } - break; - case T_NullTest: - { - NullTest *nt = (NullTest *)node; - - APP_JUMB(nt->nulltesttype); - JumbleExpr(jstate, (Node *)nt->arg); - } - break; - case T_BooleanTest: - { - BooleanTest *bt = (BooleanTest *)node; - - APP_JUMB(bt->booltesttype); - JumbleExpr(jstate, (Node *)bt->arg); - } - break; - case T_CoerceToDomain: - { - CoerceToDomain *cd = (CoerceToDomain *)node; - - APP_JUMB(cd->resulttype); - JumbleExpr(jstate, (Node *)cd->arg); - } - break; - case T_CoerceToDomainValue: - { - CoerceToDomainValue *cdv = (CoerceToDomainValue *)node; - - APP_JUMB(cdv->typeId); - } - break; - case T_SetToDefault: - { - SetToDefault *sd = (SetToDefault *)node; - - APP_JUMB(sd->typeId); - } - break; - case T_CurrentOfExpr: - { - CurrentOfExpr *ce = (CurrentOfExpr *)node; - - APP_JUMB(ce->cvarno); - if (ce->cursor_name) - APP_JUMB_STRING(ce->cursor_name); - APP_JUMB(ce->cursor_param); - } - break; - case T_TargetEntry: - { - TargetEntry *tle = (TargetEntry *)node; - - APP_JUMB(tle->resno); - APP_JUMB(tle->ressortgroupref); - JumbleExpr(jstate, (Node *)tle->expr); - } - break; - case T_RangeTblRef: - { - RangeTblRef *rtr = (RangeTblRef *)node; - - APP_JUMB(rtr->rtindex); - } - break; - case T_JoinExpr: - { - JoinExpr *join = (JoinExpr *)node; - - APP_JUMB(join->jointype); - APP_JUMB(join->isNatural); - APP_JUMB(join->rtindex); - JumbleExpr(jstate, join->larg); - JumbleExpr(jstate, join->rarg); - JumbleExpr(jstate, join->quals); - } - break; - case T_FromExpr: - { - FromExpr *from = (FromExpr *)node; - - JumbleExpr(jstate, (Node *)from->fromlist); - JumbleExpr(jstate, from->quals); - } - break; - case T_List: - foreach (temp, (List *)node) - { - JumbleExpr(jstate, (Node *)lfirst(temp)); - } - break; - case T_SortGroupClause: - { - SortGroupClause *sgc = (SortGroupClause *)node; - - APP_JUMB(sgc->tleSortGroupRef); - APP_JUMB(sgc->eqop); - APP_JUMB(sgc->sortop); - APP_JUMB(sgc->nulls_first); - } - break; - case T_WindowClause: - { - WindowClause *wc = (WindowClause *)node; - - APP_JUMB(wc->winref); - APP_JUMB(wc->frameOptions); - JumbleExpr(jstate, (Node *)wc->partitionClause); - JumbleExpr(jstate, (Node *)wc->orderClause); - JumbleExpr(jstate, wc->startOffset); - JumbleExpr(jstate, wc->endOffset); - } - break; - case T_CommonTableExpr: - { - CommonTableExpr *cte = (CommonTableExpr *)node; - - /* we store the string name because RTE_CTE RTEs need it */ - APP_JUMB_STRING(cte->ctename); - JumbleQuery(jstate, (Query *)cte->ctequery); - } - break; - case T_SetOperationStmt: - { - SetOperationStmt *setop = (SetOperationStmt *)node; - - APP_JUMB(setop->op); - APP_JUMB(setop->all); - JumbleExpr(jstate, setop->larg); - JumbleExpr(jstate, setop->rarg); - } - break; - case T_RangeTblFunction: - { - RangeTblFunction *rtfunc = (RangeTblFunction *)node; - - JumbleExpr(jstate, rtfunc->funcexpr); - } - break; - /* GPDB nodes */ - case T_GroupingClause: - { - GroupingClause *grpnode = (GroupingClause *)node; - - JumbleExpr(jstate, (Node *)grpnode->groupsets); - } - break; - case T_GroupingFunc: - { - GroupingFunc *grpnode = (GroupingFunc *)node; - - JumbleExpr(jstate, (Node *)grpnode->args); - } - break; - case T_Grouping: - case T_GroupId: - case T_Integer: - case T_Value: - // TODO:seems like nothing to do with it - break; - /* GPDB-only additions, nothing to do */ - case T_PartitionBy: - case T_PartitionElem: - case T_PartitionRangeItem: - case T_PartitionBoundSpec: - case T_PartitionSpec: - case T_PartitionValuesSpec: - case T_AlterPartitionId: - case T_AlterPartitionCmd: - case T_InheritPartitionCmd: - case T_CreateFileSpaceStmt: - case T_FileSpaceEntry: - case T_DropFileSpaceStmt: - case T_TableValueExpr: - case T_DenyLoginInterval: - case T_DenyLoginPoint: - case T_AlterTypeStmt: - case T_SetDistributionCmd: - case T_ExpandStmtSpec: - break; - default: - /* Only a warning, since we can stumble along anyway */ - ereport(WARNING, (errmsg("unrecognized node type: %d", - (int)nodeTag(node)))); - break; - } -} - -/* - * Record location of constant within query string of query tree - * that is currently being walked. - */ -static void -RecordConstLocation(pgssJumbleState *jstate, int location) -{ - /* -1 indicates unknown or undefined location */ - if (location >= 0) - { - /* enlarge array if needed */ - if (jstate->clocations_count >= jstate->clocations_buf_size) - { - jstate->clocations_buf_size *= 2; - jstate->clocations = (pgssLocationLen *) - repalloc(jstate->clocations, - jstate->clocations_buf_size * - sizeof(pgssLocationLen)); - } - jstate->clocations[jstate->clocations_count].location = location; - /* initialize lengths to -1 to simplify fill_in_constant_lengths */ - jstate->clocations[jstate->clocations_count].length = -1; - jstate->clocations_count++; - } + /* NO-OP */ } /* check if token should be replaced by substitute varable */ @@ -768,60 +127,13 @@ gen_normplan(const char *execution_plan) } /* - * Post-parse-analysis hook: mark query with a queryId - */ -void pgss_post_parse_analyze(ParseState *pstate, Query *query) -{ - pgssJumbleState jstate; - - if (prev_post_parse_analyze_hook) - prev_post_parse_analyze_hook(pstate, query); - - /* Assert we didn't do this already */ - Assert(query->queryId == 0); - - /* - * Utility statements get queryId zero. We do this even in cases where - * the statement contains an optimizable statement for which a queryId - * could be derived (such as EXPLAIN or DECLARE CURSOR). For such cases, - * runtime control will first go through ProcessUtility and then the - * executor, and we don't want the executor hooks to do anything, since we - * are already measuring the statement's costs at the utility level. - */ - if (query->utilityStmt) - { - query->queryId = 0; - return; - } - - /* Set up workspace for query jumbling */ - jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); - jstate.jumble_len = 0; - jstate.clocations_buf_size = 32; - jstate.clocations = (pgssLocationLen *) - palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); - jstate.clocations_count = 0; - - /* Compute query ID and mark the Query node with it */ - JumbleQuery(&jstate, query); - query->queryId = hash_any(jstate.jumble, jstate.jumble_len); - - /* - * If we are unlucky enough to get a hash of zero, use 1 instead, to - * prevent confusion with the utility-statement case. - */ - if (query->queryId == 0) - query->queryId = 1; -} - -/* - * comp_location: comparator for qsorting pgssLocationLen structs by location + * comp_location: comparator for qsorting LocationLen structs by location */ static int comp_location(const void *a, const void *b) { - int l = ((const pgssLocationLen *) a)->location; - int r = ((const pgssLocationLen *) b)->location; + int l = ((const LocationLen *) a)->location; + int r = ((const LocationLen *) b)->location; if (l < r) return -1; @@ -854,9 +166,9 @@ comp_location(const void *a, const void *b) * reason for a constant to start with a '-'. */ static void -fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) +fill_in_constant_lengths(JumbleState *jstate, const char *query) { - pgssLocationLen *locs; + LocationLen *locs; core_yyscan_t yyscanner; core_yy_extra_type yyextra; core_YYSTYPE yylval; @@ -870,14 +182,14 @@ fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) */ if (jstate->clocations_count > 1) qsort(jstate->clocations, jstate->clocations_count, - sizeof(pgssLocationLen), comp_location); + sizeof(LocationLen), comp_location); locs = jstate->clocations; /* initialize the flex scanner --- should match raw_parser() */ yyscanner = scanner_init(query, &yyextra, - ScanKeywords, - NumScanKeywords); + &ScanKeywords, + ScanKeywordTokens); /* Search for each constant, in sequence */ for (i = 0; i < jstate->clocations_count; i++) @@ -957,7 +269,7 @@ fill_in_constant_lengths(pgssJumbleState *jstate, const char *query) * Returns a palloc'd string. */ static char * -generate_normalized_query(pgssJumbleState *jstate, const char *query, +generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding) { char *norm_query; @@ -1027,12 +339,12 @@ char *gen_normquery(const char *query) if (!query) { return NULL; } - pgssJumbleState jstate; + JumbleState jstate; jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); jstate.jumble_len = 0; jstate.clocations_buf_size = 32; - jstate.clocations = (pgssLocationLen *) - palloc(jstate.clocations_buf_size * sizeof(pgssLocationLen)); + jstate.clocations = (LocationLen *) + palloc(jstate.clocations_buf_size * sizeof(LocationLen)); jstate.clocations_count = 0; int query_len = strlen(query); return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 9db73638b24..27fd0e04b26 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,5 +1,6 @@ #include "postgres.h" #include "cdb/cdbvars.h" +#include "funcapi.h" #include "utils/builtins.h" #include "hook_wrappers.h" @@ -26,8 +27,15 @@ void _PG_fini(void) { } Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { - yagp_functions_reset(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + yagp_functions_reset(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } Datum yagp_stat_messages(PG_FUNCTION_ARGS) { @@ -35,11 +43,25 @@ Datum yagp_stat_messages(PG_FUNCTION_ARGS) { } Datum yagp_init_log(PG_FUNCTION_ARGS) { - init_log(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + init_log(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } Datum yagp_truncate_log(PG_FUNCTION_ARGS) { - truncate_log(); - PG_RETURN_VOID(); + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + truncate_log(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } diff --git a/yagp_hooks_collector--1.0--1.1.sql b/yagp_hooks_collector--1.0--1.1.sql index 959d4f235d1..8684ca73915 100644 --- a/yagp_hooks_collector--1.0--1.1.sql +++ b/yagp_hooks_collector--1.0--1.1.sql @@ -23,17 +23,17 @@ DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); -- Recreate functions and view in new schema. CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT yagpcc.__stat_messages_reset_f_on_master(); @@ -75,12 +75,12 @@ ORDER BY segid; -- Create new objects. CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; @@ -95,17 +95,17 @@ CREATE VIEW yagpcc.log AS ORDER BY tmid, ssid, ccnt; CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.truncate_log() -RETURNS void AS $$ +RETURNS SETOF void AS $$ BEGIN PERFORM yagpcc.__truncate_log_on_master(); PERFORM yagpcc.__truncate_log_on_segments(); diff --git a/yagp_hooks_collector--1.0.sql b/yagp_hooks_collector--1.0.sql index 7ab4e1b2fb7..270cab92382 100644 --- a/yagp_hooks_collector--1.0.sql +++ b/yagp_hooks_collector--1.0.sql @@ -4,17 +4,17 @@ \echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagp_stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT __yagp_stat_messages_reset_f_on_master(); diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql index 657720a88f2..e0e94b51493 100644 --- a/yagp_hooks_collector--1.1.sql +++ b/yagp_hooks_collector--1.1.sql @@ -6,17 +6,17 @@ CREATE SCHEMA yagpcc; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' LANGUAGE C EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS void +RETURNS SETOF void AS $$ SELECT yagpcc.__stat_messages_reset_f_on_master(); @@ -57,12 +57,12 @@ CREATE VIEW yagpcc.stat_messages AS ORDER BY segid; CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_init_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; @@ -77,17 +77,17 @@ CREATE VIEW yagpcc.log AS ORDER BY tmid, ssid, ccnt; CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS void +RETURNS SETOF void AS 'MODULE_PATHNAME', 'yagp_truncate_log' LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; CREATE FUNCTION yagpcc.truncate_log() -RETURNS void AS $$ +RETURNS SETOF void AS $$ BEGIN PERFORM yagpcc.__truncate_log_on_master(); PERFORM yagpcc.__truncate_log_on_segments(); From 5a2cc69de4cd4da90fecee599c0918f71bc5d89e Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 11:13:00 +0300 Subject: [PATCH 093/167] [yagp_hooks_collector] Add --with-yagp-hooks-collector configure option and CI Add configure.ac option with protobuf dependency. Add CI test configuration. Change env script from greenplum_path.sh to cloudberry-env.sh. --- .github/workflows/build-cloudberry.yml | 32 ++++++++- configure | 5 +- configure.ac | 7 ++ .../scripts/configure-cloudberry.sh | 4 +- expected/yagp_cursors.out | 10 +-- expected/yagp_dist.out | 2 + expected/yagp_select.out | 2 + expected/yagp_utf8_trim.out | 2 + expected/yagp_utility.out | 72 ++++++++++--------- gpcontrib/yagp_hooks_collector/Makefile | 41 +++++++++++ sql/yagp_cursors.sql | 2 + sql/yagp_dist.sql | 2 + sql/yagp_select.sql | 2 + sql/yagp_utf8_trim.sql | 2 + sql/yagp_utility.sql | 2 + 15 files changed, 145 insertions(+), 42 deletions(-) create mode 100644 gpcontrib/yagp_hooks_collector/Makefile diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 80dae61a5be..6b956954700 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -271,6 +271,10 @@ jobs: }, "enable_core_check":false }, + {"test":"gpcontrib-yagp-hooks-collector", + "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"], + "extension":"yagp_hooks_collector" + }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] }, @@ -537,10 +541,11 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1405,6 +1410,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: /usr/local/cloudberry-db shell: bash {0} run: | set -o pipefail @@ -1434,6 +1440,30 @@ jobs: PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" fi + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + yagp_hooks_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating yagp_hooks_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" fi diff --git a/configure b/configure index c99ab5563f1..e2d0232ead9 100755 --- a/configure +++ b/configure @@ -723,6 +723,7 @@ with_libcurl with_rt with_zstd with_yezzey +with_yagp_hooks_collector with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -946,6 +947,7 @@ with_zstd with_diskquota with_gp_stats_collector with_yezzey +with_yagp_hooks_collector with_rt with_libcurl with_apr_config @@ -11247,7 +11249,6 @@ fi # - # Check whether --with-gp-stats-collector was given. if test "${with_gp_stats_collector+set}" = set; then : withval=$with_gp_stats_collector; @@ -11264,6 +11265,7 @@ if test "${with_gp_stats_collector+set}" = set; then : esac else + with_gp_stats_collector=no fi @@ -11288,6 +11290,7 @@ if test "${with_yezzey+set}" = set; then : esac else + with_yezzey=no fi diff --git a/configure.ac b/configure.ac index 308e0872f07..a2a69b069f9 100644 --- a/configure.ac +++ b/configure.ac @@ -1368,6 +1368,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# yagp_hooks_collector +# +PGAC_ARG_BOOL(with, yagp_hooks_collector, no, + [build with YAGP hooks collector extension]) +AC_SUBST(with_yagp_hooks_collector) + if test "$with_zstd" = yes; then dnl zstd_errors.h was renamed from error_public.h in v1.4.0 PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.4.0]) diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 2d7ad04aed8..d30a0b794f0 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -53,6 +53,7 @@ # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) +# CONFIGURE_EXTRA_OPTS - Args to pass to configure command # ENABLE_DEBUG - Enable debug build options (true/false, defaults to # false) # @@ -177,7 +178,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-uuid=e2fs \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib \ + ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 log_section_end "Configure" # Capture version information diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index d251ddd3e1c..46e124df5e8 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -25,7 +26,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT @@ -54,7 +55,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -86,7 +87,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -129,7 +130,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT @@ -159,3 +160,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_dist.out b/expected/yagp_dist.out index 5fd5ea5fb3e..3b1e3504923 100644 --- a/expected/yagp_dist.out +++ b/expected/yagp_dist.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -171,3 +172,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_select.out b/expected/yagp_select.out index b6e18dc862f..af08f2d1def 100644 --- a/expected/yagp_select.out +++ b/expected/yagp_select.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -132,3 +133,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/expected/yagp_utf8_trim.out b/expected/yagp_utf8_trim.out index 194ee6b3609..9de126dd882 100644 --- a/expected/yagp_utf8_trim.out +++ b/expected/yagp_utf8_trim.out @@ -7,6 +7,7 @@ RETURNS TEXT AS $$ ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; -- Test 1: 1 byte chars SET yagpcc.max_text_size to 19; @@ -63,4 +64,5 @@ DROP FUNCTION get_marked_query(TEXT); RESET yagpcc.max_text_size; RESET yagpcc.logging_mode; RESET yagpcc.enable; +RESET yagpcc.ignored_users_list; DROP EXTENSION yagp_hooks_collector; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 057f7d7a556..0a77859d8d4 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -12,6 +12,7 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -26,7 +27,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT @@ -83,7 +84,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT @@ -113,26 +114,26 @@ BEGIN; ROLLBACK; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------+--------------------- - -1 | | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+-----------------------------------+--------------------- + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -153,7 +154,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT @@ -176,16 +177,16 @@ COPY (SELECT 1) TO STDOUT; DROP TABLE copy_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+-----------------------------------+--------------------- + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -203,7 +204,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT @@ -226,7 +227,7 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- - -1 | | QUERY_STATUS_DONE + -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT @@ -244,3 +245,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile new file mode 100644 index 00000000000..be46eb7149c --- /dev/null +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -0,0 +1,41 @@ +MODULE_big = yagp_hooks_collector +EXTENSION = yagp_hooks_collector +DATA = $(wildcard *--*.sql) +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility + +PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service +PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) + +C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) +CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) +OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) + +override CXXFLAGS = -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ + -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ + -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ + -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD + +PG_CXXFLAGS += -Isrc -Iinclude +SHLIB_LINK += -lprotobuf -lpthread -lstdc++ +EXTRA_CLEAN = src/protos + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = gpcontrib/yagp_hooks_collector +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto + @mkdir -p src/protos + sed -i 's/optional //g' $^ + sed -i 's|cloud/mdb/yagpcc/api/proto/common/|protos/|g' $^ + protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ + mv src/protos/$*.pb.cc src/protos/$*.pb.cpp + +$(CPP_OBJS): src/protos/yagpcc_metrics.pb.h src/protos/yagpcc_plan.pb.h src/protos/yagpcc_set_service.pb.h +src/protos/yagpcc_set_service.pb.o: src/protos/yagpcc_metrics.pb.h diff --git a/sql/yagp_cursors.sql b/sql/yagp_cursors.sql index 5d5bde58110..f56351e0d43 100644 --- a/sql/yagp_cursors.sql +++ b/sql/yagp_cursors.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -81,3 +82,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_dist.sql b/sql/yagp_dist.sql index b837ef05335..d5519d0cd96 100644 --- a/sql/yagp_dist.sql +++ b/sql/yagp_dist.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -84,3 +85,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_select.sql b/sql/yagp_select.sql index 4038c6b7b63..90e972ae4c1 100644 --- a/sql/yagp_select.sql +++ b/sql/yagp_select.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.report_nested_queries TO TRUE; SET yagpcc.enable_utility TO FALSE; @@ -65,3 +66,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; diff --git a/sql/yagp_utf8_trim.sql b/sql/yagp_utf8_trim.sql index c0fdcce24a5..c3053e4af0c 100644 --- a/sql/yagp_utf8_trim.sql +++ b/sql/yagp_utf8_trim.sql @@ -9,6 +9,7 @@ RETURNS TEXT AS $$ LIMIT 1 $$ LANGUAGE sql VOLATILE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; -- Test 1: 1 byte chars @@ -39,5 +40,6 @@ DROP FUNCTION get_marked_query(TEXT); RESET yagpcc.max_text_size; RESET yagpcc.logging_mode; RESET yagpcc.enable; +RESET yagpcc.ignored_users_list; DROP EXTENSION yagp_hooks_collector; diff --git a/sql/yagp_utility.sql b/sql/yagp_utility.sql index b4cca6f5421..cf9c1d253d0 100644 --- a/sql/yagp_utility.sql +++ b/sql/yagp_utility.sql @@ -14,6 +14,7 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +SET yagpcc.ignored_users_list TO ''; SET yagpcc.enable TO TRUE; SET yagpcc.enable_utility TO TRUE; SET yagpcc.report_nested_queries TO TRUE; @@ -131,3 +132,4 @@ DROP EXTENSION yagp_hooks_collector; RESET yagpcc.enable; RESET yagpcc.report_nested_queries; RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; From f0f29518a6a5c945b0924cce47093feb6f092c9a Mon Sep 17 00:00:00 2001 From: NJrslv Date: Mon, 19 Jan 2026 17:05:08 +0300 Subject: [PATCH 094/167] [yagp_hooks_collector] Add consistent GUC filtering and submit/done hook callsites Cache GUC values at SUBMIT so filtering criteria remain consistent across the full query lifecycle. Add query_info_collect_hook calls in ExecCreateTableAs, refresh_matview_datafill, and PortalCleanup. Correct tokens from gram.y. --- expected/yagp_cursors.out | 8 +- expected/yagp_guc_cache.out | 57 ++++++++++++ expected/yagp_utility.out | 72 +++++++-------- gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_guc_cache.sql | 43 +++++++++ src/Config.cpp | 90 +++++++++---------- src/Config.h | 49 +++++++--- src/EventSender.cpp | 68 ++++++++------ src/EventSender.h | 10 ++- src/PgUtils.cpp | 14 --- src/PgUtils.h | 3 - src/ProtoUtils.cpp | 28 +++--- src/ProtoUtils.h | 13 ++- src/UDSConnector.cpp | 5 +- src/UDSConnector.h | 6 +- src/backend/commands/createas.c | 8 +- src/backend/commands/matview.c | 5 ++ src/backend/commands/portalcmds.c | 5 ++ src/hook_wrappers.cpp | 2 +- src/log/LogOps.cpp | 6 +- .../pg_stat_statements_ya_parser.c | 14 +-- 21 files changed, 325 insertions(+), 183 deletions(-) create mode 100644 expected/yagp_guc_cache.out create mode 100644 sql/yagp_guc_cache.sql diff --git a/expected/yagp_cursors.out b/expected/yagp_cursors.out index 46e124df5e8..df12e3e1b66 100644 --- a/expected/yagp_cursors.out +++ b/expected/yagp_cursors.out @@ -26,7 +26,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT @@ -36,6 +35,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -55,7 +55,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -69,6 +68,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (14 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -87,7 +87,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT @@ -99,6 +98,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | ROLLBACK; | QUERY_STATUS_SUBMIT -1 | ROLLBACK; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (12 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -130,7 +130,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | BEGIN; | QUERY_STATUS_SUBMIT -1 | BEGIN; | QUERY_STATUS_DONE -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT @@ -148,6 +147,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; diff --git a/expected/yagp_guc_cache.out b/expected/yagp_guc_cache.out new file mode 100644 index 00000000000..3085cfa42e1 --- /dev/null +++ b/expected/yagp_guc_cache.out @@ -0,0 +1,57 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +SELECT yagpcc.truncate_log(); +-- end_ignore +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM yagpcc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.logging_mode TO 'TBL'; +-- SET below disables utility logging and DONE must still be logged. +SET yagpcc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_DONE +(2 rows) + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('yagpcc.ignored_users_list', current_user, false); + set_config +------------ + gpadmin +(1 row) + +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_START + QUERY_STATUS_END + QUERY_STATUS_DONE +(4 rows) + +DROP FUNCTION print_last_query(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +RESET yagpcc.logging_mode; diff --git a/expected/yagp_utility.out b/expected/yagp_utility.out index 0a77859d8d4..7df1d2816eb 100644 --- a/expected/yagp_utility.out +++ b/expected/yagp_utility.out @@ -27,7 +27,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT @@ -37,6 +36,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT -1 | DROP TABLE test_table; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -55,7 +55,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT | DISTRIBUTED BY (a) +| | PARTITION BY RANGE (a) +| @@ -67,6 +66,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -84,7 +84,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT @@ -94,6 +93,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (10 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -114,26 +114,26 @@ BEGIN; ROLLBACK; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+----------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (18 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -154,12 +154,12 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -177,16 +177,16 @@ COPY (SELECT 1) TO STDOUT; DROP TABLE copy_test; RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -204,7 +204,6 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT @@ -212,6 +211,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (8 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; @@ -227,12 +227,12 @@ RESET yagpcc.logging_mode; SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- - -1 | SET yagpcc.logging_mode to 'TBL'; | QUERY_STATUS_DONE -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE (6 rows) SELECT yagpcc.truncate_log() IS NOT NULL AS t; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index be46eb7149c..79f5401c8d1 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_guc_cache.sql b/sql/yagp_guc_cache.sql new file mode 100644 index 00000000000..9e6de69d61e --- /dev/null +++ b/sql/yagp_guc_cache.sql @@ -0,0 +1,43 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +SELECT yagpcc.truncate_log(); +-- end_ignore + +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM yagpcc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; + +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.logging_mode TO 'TBL'; + +-- SET below disables utility logging and DONE must still be logged. +SET yagpcc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('yagpcc.ignored_users_list', current_user, false); +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); + +DROP FUNCTION print_last_query(text); +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +RESET yagpcc.logging_mode; diff --git a/src/Config.cpp b/src/Config.cpp index dbd7e25b483..4fb58677018 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -27,45 +27,13 @@ static const struct config_enum_entry logging_mode_options[] = { {"tbl", LOG_MODE_TBL, false}, {NULL, 0, false}}; -static std::unique_ptr> ignored_users_set = - nullptr; static bool ignored_users_guc_dirty = false; -static void update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = - std::make_unique>(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { - /* syntax error in list */ - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); - } - ignored_users_set = std::move(new_ignored_users_set); -} - static void assign_ignored_users_hook(const char *, void *) { ignored_users_guc_dirty = true; } -void Config::init() { +void Config::init_gucs() { DefineCustomStringVariable( "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, @@ -128,22 +96,40 @@ void Config::init() { GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } -std::string Config::uds_path() { return guc_uds_path; } -bool Config::enable_analyze() { return guc_enable_analyze; } -bool Config::enable_cdbstats() { return guc_enable_cdbstats; } -bool Config::enable_collector() { return guc_enable_collector; } -bool Config::enable_utility() { return guc_enable_utility; } -bool Config::report_nested_queries() { return guc_report_nested_queries; } -size_t Config::max_text_size() { return guc_max_text_size; } -size_t Config::max_plan_size() { return guc_max_plan_size * 1024; } -int Config::min_analyze_time() { return guc_min_analyze_time; }; -int Config::logging_mode() { return guc_logging_mode; } - -bool Config::filter_user(std::string username) { - if (!ignored_users_set) { +void Config::update_ignored_users(const char *new_guc_ignored_users) { + auto new_ignored_users_set = std::make_unique(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { + /* Need a modifiable copy of string */ + char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + /* syntax error in list */ + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter yagpcc.ignored_users_list"))); + return; + } + foreach (l, elemlist) { + new_ignored_users_set->insert((char *)lfirst(l)); + } + ya_gpdb::pfree(rawstring); + ya_gpdb::list_free(elemlist); + } + ignored_users_ = std::move(new_ignored_users_set); +} + +bool Config::filter_user(const std::string &username) const { + if (!ignored_users_) { return true; } - return ignored_users_set->find(username) != ignored_users_set->end(); + return ignored_users_->find(username) != ignored_users_->end(); } void Config::sync() { @@ -151,4 +137,14 @@ void Config::sync() { update_ignored_users(guc_ignored_users); ignored_users_guc_dirty = false; } + uds_path_ = guc_uds_path; + enable_analyze_ = guc_enable_analyze; + enable_cdbstats_ = guc_enable_cdbstats; + enable_collector_ = guc_enable_collector; + enable_utility_ = guc_enable_utility; + report_nested_queries_ = guc_report_nested_queries; + max_text_size_ = static_cast(guc_max_text_size); + max_plan_size_ = static_cast(guc_max_plan_size); + min_analyze_time_ = guc_min_analyze_time; + logging_mode_ = guc_logging_mode; } diff --git a/src/Config.h b/src/Config.h index 7501c727a44..b4a393b0383 100644 --- a/src/Config.h +++ b/src/Config.h @@ -1,23 +1,44 @@ #pragma once +#include #include +#include #define LOG_MODE_UDS 0 #define LOG_MODE_TBL 1 +using IgnoredUsers = std::unordered_set; + class Config { public: - static void init(); - static std::string uds_path(); - static bool enable_analyze(); - static bool enable_cdbstats(); - static bool enable_collector(); - static bool enable_utility(); - static bool filter_user(std::string username); - static bool report_nested_queries(); - static size_t max_text_size(); - static size_t max_plan_size(); - static int min_analyze_time(); - static int logging_mode(); - static void sync(); -}; \ No newline at end of file + static void init_gucs(); + + void sync(); + + const std::string &uds_path() const { return uds_path_; } + bool enable_analyze() const { return enable_analyze_; } + bool enable_cdbstats() const { return enable_cdbstats_; } + bool enable_collector() const { return enable_collector_; } + bool enable_utility() const { return enable_utility_; } + bool report_nested_queries() const { return report_nested_queries_; } + size_t max_text_size() const { return max_text_size_; } + size_t max_plan_size() const { return max_plan_size_ * 1024; } + int min_analyze_time() const { return min_analyze_time_; } + int logging_mode() const { return logging_mode_; } + bool filter_user(const std::string &username) const; + +private: + void update_ignored_users(const char *new_guc_ignored_users); + + std::unique_ptr ignored_users_; + std::string uds_path_; + bool enable_analyze_; + bool enable_cdbstats_; + bool enable_collector_; + bool enable_utility_; + bool report_nested_queries_; + size_t max_text_size_; + size_t max_plan_size_; + int min_analyze_time_; + int logging_mode_; +}; diff --git a/src/EventSender.cpp b/src/EventSender.cpp index d638d275548..853a0c43fb9 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,4 +1,3 @@ -#include "Config.h" #include "UDSConnector.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" @@ -22,10 +21,8 @@ extern "C" { #include "ProtoUtils.h" #define need_collect_analyze() \ - (Gp_role == GP_ROLE_DISPATCH && Config::min_analyze_time() >= 0 && \ - Config::enable_analyze()) - -static bool enable_utility = Config::enable_utility(); + (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ + config.enable_analyze()) bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool utility) { @@ -38,16 +35,16 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, switch (state) { case QueryState::SUBMIT: - // Cache enable_utility at SUBMIT to ensure consistent behavior at DONE. - // Without caching, a query that sets enable_utility to false from true - // would be accepted at SUBMIT (guc is true) but rejected at DONE (guc - // is false), causing a leak. - enable_utility = Config::enable_utility(); - if (utility && enable_utility == false) { + // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent + // states. Without caching, a query that unsets/sets filtering GUCs would + // see different filter criteria at DONE, because at SUBMIT the query was + // not executed yet, causing DONE to be skipped/added. + config.sync(); + + if (utility && !config.enable_utility()) { return false; } - // Sync config in case current query changes it. - Config::sync(); + // Register qkey for a nested query we won't report, // so we can detect nesting_level > 0 and skip reporting at end/done. if (!need_report_nested_query() && nesting_level > 0) { @@ -65,7 +62,7 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, } break; case QueryState::DONE: - if (utility && enable_utility == false) { + if (utility && !config.enable_utility()) { return false; } default: @@ -85,9 +82,9 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, bool utility) { bool clear_big_fields = false; - switch (Config::logging_mode()) { + switch (config.logging_mode()) { case LOG_MODE_UDS: - clear_big_fields = UDSConnector::report_query(req, event); + clear_big_fields = UDSConnector::report_query(req, event, config); break; case LOG_MODE_TBL: ya_gpdb::insert_log(req, utility); @@ -135,12 +132,12 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { return; } - if (Gp_role == GP_ROLE_DISPATCH && Config::enable_analyze() && + if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { query_desc->instrument_options |= INSTRUMENT_BUFFERS; query_desc->instrument_options |= INSTRUMENT_ROWS; query_desc->instrument_options |= INSTRUMENT_TIMER; - if (Config::enable_cdbstats()) { + if (config.enable_cdbstats()) { query_desc->instrument_options |= INSTRUMENT_CDB; if (!query_desc->showstatctx) { instr_time starttime; @@ -161,7 +158,7 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { auto query_msg = query.message.get(); *query_msg->mutable_start_time() = current_ts(); update_query_state(query, QueryState::START, false /* utility */); - set_query_plan(query_msg, query_desc); + set_query_plan(query_msg, query_desc, config); if (need_collect_analyze()) { // Set up to track total elapsed time during query run. // Make sure the space is allocated in the per-query @@ -214,7 +211,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { set_query_info(query_msg); set_qi_nesting_level(query_msg, nesting_level); set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc); + set_query_text(query_msg, query_desc, config); if (log_query_req(*query_msg, "submit", utility)) { clear_big_fields(query_msg); } @@ -271,8 +268,8 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, ereport(DEBUG3, (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); } else { - set_qi_error_message(query_msg, - error_flushed ? edata->message : elog_message()); + set_qi_error_message( + query_msg, error_flushed ? edata->message : elog_message(), config); } } if (prev_state == START) { @@ -331,8 +328,8 @@ void EventSender::ic_metrics_collect() { if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { return; } - if (!proto_verified || gp_command_count == 0 || !Config::enable_collector() || - Config::filter_user(get_user_name())) { + if (!proto_verified || gp_command_count == 0 || !config.enable_collector() || + config.filter_user(get_user_name())) { return; } // we also would like to know nesting level here and filter queries BUT we @@ -374,15 +371,18 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { ya_gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; - if (ms >= Config::min_analyze_time()) { + if (ms >= config.min_analyze_time()) { auto &query = get_query(query_desc); auto *query_msg = query.message.get(); - set_analyze_plan_text(query_desc, query_msg); + set_analyze_plan_text(query_desc, query_msg, config); } } EventSender::EventSender() { - if (Config::enable_collector()) { + // Perform initial sync to get default GUC values + config.sync(); + + if (config.enable_collector()) { try { GOOGLE_PROTOBUF_VERIFY_VERSION; proto_verified = true; @@ -486,5 +486,19 @@ bool EventSender::qdesc_submitted(QueryDesc *query_desc) { return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); } +bool EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) { + return need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); +} + +bool EventSender::need_report_nested_query() { + return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; +} + +bool EventSender::filter_query(QueryDesc *query_desc) { + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !config.enable_collector() || config.filter_user(get_user_name()); +} + EventSender::QueryItem::QueryItem(QueryState st) : message(std::make_unique()), state(st) {} diff --git a/src/EventSender.h b/src/EventSender.h index 6e195eeacdf..e9acb04422b 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -14,6 +14,7 @@ extern "C" { #undef typeid #include "memory/gpdbwrappers.h" +#include "Config.h" class UDSConnector; struct QueryDesc; @@ -108,8 +109,8 @@ class EventSender { explicit QueryItem(QueryState st); }; - static bool log_query_req(const yagpcc::SetQueryReq &req, - const std::string &event, bool utility); + bool log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, + bool utility); bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); void update_query_state(QueryItem &query, QueryState new_state, bool utility, bool success = true); @@ -123,6 +124,9 @@ class EventSender { QueryMetricsStatus status, ErrorData *edata = NULL); void update_nested_counters(QueryDesc *query_desc); bool qdesc_submitted(QueryDesc *query_desc); + bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); + bool need_report_nested_query(); + bool filter_query(QueryDesc *query_desc); bool proto_verified = false; int nesting_level = 0; @@ -132,4 +136,6 @@ class EventSender { ICStatistics ic_statistics; #endif std::unordered_map queries; + + Config config; }; \ No newline at end of file diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 96f46429643..7e53abdabbf 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -65,17 +65,3 @@ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { } return query_desc->yagp_query_key->nesting_level == 0; } - -bool nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); -} - -bool need_report_nested_query() { - return Config::report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; -} - -bool filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !Config::enable_collector() || Config::filter_user(get_user_name()); -} diff --git a/src/PgUtils.h b/src/PgUtils.h index 02f084c597a..e9715ce10f4 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -9,6 +9,3 @@ std::string get_user_name(); std::string get_db_name(); std::string get_rg_name(); bool is_top_level_query(QueryDesc *query_desc, int nesting_level); -bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); -bool need_report_nested_query(); -bool filter_query(QueryDesc *query_desc); diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index aa8632477f5..8ebbe19e289 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -82,7 +82,8 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { return std::string(str, cut_pos); } -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER @@ -93,10 +94,10 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { ExplainState es = ya_gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, - Config::max_plan_size()); + config.max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, Config::max_plan_size()); + norm_plan->data, norm_plan->len, config.max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); qi->set_query_id(query_desc->plannedstmt->queryId); @@ -107,15 +108,16 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { } } -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc) { +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), - Config::max_text_size()); + config.max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), Config::max_text_size()); + norm_query, strlen(norm_query), config.max_text_size()); } } @@ -150,10 +152,11 @@ void set_qi_slice_id(yagpcc::SetQueryReq *req) { aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg) { +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, + const Config &config) { auto aqi = req->mutable_add_info(); *aqi->mutable_error_message() = - trim_str_shrink_utf8(err_msg, strlen(err_msg), Config::max_text_size()); + trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, @@ -257,7 +260,8 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, + const Config &config) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. if (!IsTransactionState() || !query_desc->plannedstmt) { @@ -266,15 +270,15 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req) { MemoryContext oldcxt = ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); ExplainState es = ya_gpdb::get_analyze_state( - query_desc, query_desc->instrument_options && Config::enable_analyze()); + query_desc, query_desc->instrument_options && config.enable_analyze()); ya_gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { es.str->data[--es.str->len] = '\0'; } - auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, - Config::max_plan_size()); + auto trimmed_analyze = + trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); ya_gpdb::pfree(es.str->data); } diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 725a634f765..37b7e4a8a29 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -4,19 +4,24 @@ struct QueryDesc; struct ICStatistics; +class Config; google::protobuf::Timestamp current_ts(); -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc); -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc); +void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config); +void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config); void clear_big_fields(yagpcc::SetQueryReq *req); void set_query_info(yagpcc::SetQueryReq *req); void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg); +void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, + const Config &config); void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); void set_ic_stats(yagpcc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message); \ No newline at end of file +void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message, + const Config &config); diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index a7eaed539f7..74fd57a3ac0 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -25,10 +25,11 @@ static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, } bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, - const std::string &event) { + const std::string &event, + const Config &config) { sockaddr_un address; address.sun_family = AF_UNIX; - std::string uds_path = Config::uds_path(); + const std::string &uds_path = config.uds_path(); if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); YagpStat::report_error(); diff --git a/src/UDSConnector.h b/src/UDSConnector.h index f0dfcb77a3f..9483407159d 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -2,8 +2,10 @@ #include "protos/yagpcc_set_service.pb.h" +class Config; + class UDSConnector { public: bool static report_query(const yagpcc::SetQueryReq &req, - const std::string &event); -}; \ No newline at end of file + const std::string &event, const Config &config); +}; diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 6822032fe0d..a3d2f155fd8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -478,10 +478,6 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, dest, params, queryEnv, 0); } - /* GPDB hook for collecting query info */ - if (query_info_collect_hook) - (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); - if (into->skipData) { /* @@ -495,6 +491,10 @@ ExecCreateTableAs(ParseState *pstate, CreateTableAsStmt *stmt, } else { + /* GPDB hook for collecting query info */ + if (query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); + check_and_unassign_from_resgroup(queryDesc->plannedstmt); queryDesc->plannedstmt->query_mem = ResourceManagerGetQueryMemoryLimit(queryDesc->plannedstmt); diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 1555ea9d334..dc8efd4d892 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -63,6 +63,7 @@ #include "tcop/tcopprot.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/metrics_utils.h" #include "utils/rel.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -842,6 +843,10 @@ refresh_matview_datafill(DestReceiver *dest, Query *query, GetActiveSnapshot(), InvalidSnapshot, dest, NULL, NULL, 0); + /* GPDB hook for collecting query info */ + if (query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_SUBMIT, queryDesc); + RestoreOidAssignments(saved_dispatch_oids); /* call ExecutorStart to prepare the plan for execution */ diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 4817c14f07d..553830e8599 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -35,6 +35,7 @@ #include "tcop/pquery.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" +#include "utils/metrics_utils.h" #include "utils/snapmgr.h" #include "cdb/cdbendpoint.h" @@ -373,6 +374,10 @@ PortalCleanup(Portal portal) FreeQueryDesc(queryDesc); CurrentResourceOwner = saveResourceOwner; + } else { + /* GPDB hook for collecting query info */ + if (queryDesc->yagp_query_key && query_info_collect_hook) + (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); } } diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 56c1da9f4f6..8cf74641c29 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -71,7 +71,7 @@ R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { } void hooks_init() { - Config::init(); + Config::init_gucs(); YagpStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; ExecutorStart_hook = ya_ExecutorStart_hook; diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index cec9e33693a..56bdf1dca62 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -38,9 +38,9 @@ void init_log() { log_relname.data() /* relname */, namespaceId /* namespace */, 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, - DescribeTuple() /* rel tuple */, NIL, RELKIND_RELATION, - RELPERSISTENCE_PERMANENT, false, false, ONCOMMIT_NOOP, - NULL /* GP Policy */, (Datum)0, false /* use_user_acl */, true, true, + DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, RELKIND_RELATION, + RELPERSISTENCE_PERMANENT, false /* shared_relation */, false /* mapped_relation */, ONCOMMIT_NOOP, + NULL /* GP Policy */, (Datum)0 /* reloptions */, false /* use_user_acl */, true /* allow_system_table_mods */, true /* is_internal */, InvalidOid /* relrewrite */, NULL /* typaddress */, false /* valid_opts */); diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index c19805ce506..54c8b2cf59f 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -17,20 +17,20 @@ #include "pg_stat_statements_ya_parser.h" -#ifndef ICONST -#define ICONST 276 -#endif #ifndef FCONST -#define FCONST 277 +#define FCONST 260 #endif #ifndef SCONST -#define SCONST 278 +#define SCONST 261 #endif #ifndef BCONST -#define BCONST 279 +#define BCONST 263 #endif #ifndef XCONST -#define XCONST 280 +#define XCONST 264 +#endif +#ifndef ICONST +#define ICONST 266 #endif static void fill_in_constant_lengths(JumbleState *jstate, const char *query); From a2cbf7c0b34191a00230fb32b135402df171f41c Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 20 Jan 2026 17:03:53 +0300 Subject: [PATCH 095/167] [yagp_hooks_collector] Add UDS round-trip test and fix send() accounting Add regression test for UDS transport. Fix send() return value: do not add -1 to total_bytes_sent on error. General refactoring. --- expected/yagp_uds.out | 42 +++++++++ gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_uds.sql | 31 +++++++ src/Config.cpp | 10 +- src/Config.h | 8 +- src/UDSConnector.cpp | 117 +++++++++++++----------- src/hook_wrappers.cpp | 96 ++++++++++++++++++- src/hook_wrappers.h | 4 + src/yagp_hooks_collector.c | 64 ++++++++++++- yagp_hooks_collector--1.1.sql | 15 +++ 10 files changed, 318 insertions(+), 71 deletions(-) create mode 100644 expected/yagp_uds.out create mode 100644 sql/yagp_uds.sql diff --git a/expected/yagp_uds.out b/expected/yagp_uds.out new file mode 100644 index 00000000000..d04929ffb4a --- /dev/null +++ b/expected/yagp_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +-- end_ignore +\set UDS_PATH '/tmp/yagpcc_test.sock' +-- Configure extension to send via UDS +SET yagpcc.uds_path TO :'UDS_PATH'; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.logging_mode TO 'UDS'; +-- Start receiver +SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT yagpcc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT yagpcc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.uds_path; +RESET yagpcc.ignored_users_list; +RESET yagpcc.enable; +RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index 79f5401c8d1..eb6541b7687 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_uds.sql b/sql/yagp_uds.sql new file mode 100644 index 00000000000..3eef697a4e7 --- /dev/null +++ b/sql/yagp_uds.sql @@ -0,0 +1,31 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +-- end_ignore + +\set UDS_PATH '/tmp/yagpcc_test.sock' + +-- Configure extension to send via UDS +SET yagpcc.uds_path TO :'UDS_PATH'; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable TO TRUE; +SET yagpcc.logging_mode TO 'UDS'; + +-- Start receiver +SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); + +-- Send +SELECT 1; + +-- Receive +SELECT yagpcc.__test_uds_receive() > 0 as received; + +-- Stop receiver +SELECT yagpcc.__test_uds_stop_server(); + +-- Cleanup +DROP EXTENSION yagp_hooks_collector; +RESET yagpcc.uds_path; +RESET yagpcc.ignored_users_list; +RESET yagpcc.enable; +RESET yagpcc.logging_mode; diff --git a/src/Config.cpp b/src/Config.cpp index 4fb58677018..2c2032ebb03 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -16,9 +16,9 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = true; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1 << 20; // in bytes (1MB) -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = 10000; // in ms +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms static int guc_logging_mode = LOG_MODE_UDS; static bool guc_enable_utility = false; @@ -143,8 +143,8 @@ void Config::sync() { enable_collector_ = guc_enable_collector; enable_utility_ = guc_enable_utility; report_nested_queries_ = guc_report_nested_queries; - max_text_size_ = static_cast(guc_max_text_size); - max_plan_size_ = static_cast(guc_max_plan_size); + max_text_size_ = guc_max_text_size; + max_plan_size_ = guc_max_plan_size; min_analyze_time_ = guc_min_analyze_time; logging_mode_ = guc_logging_mode; } diff --git a/src/Config.h b/src/Config.h index b4a393b0383..aa6b5bdc0ba 100644 --- a/src/Config.h +++ b/src/Config.h @@ -21,8 +21,8 @@ class Config { bool enable_collector() const { return enable_collector_; } bool enable_utility() const { return enable_utility_; } bool report_nested_queries() const { return report_nested_queries_; } - size_t max_text_size() const { return max_text_size_; } - size_t max_plan_size() const { return max_plan_size_ * 1024; } + int max_text_size() const { return max_text_size_; } + int max_plan_size() const { return max_plan_size_ * 1024; } int min_analyze_time() const { return min_analyze_time_; } int logging_mode() const { return logging_mode_; } bool filter_user(const std::string &username) const; @@ -37,8 +37,8 @@ class Config { bool enable_collector_; bool enable_utility_; bool report_nested_queries_; - size_t max_text_size_; - size_t max_plan_size_; + int max_text_size_; + int max_plan_size_; int min_analyze_time_; int logging_mode_; }; diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index 74fd57a3ac0..ea118fca783 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -27,66 +27,77 @@ static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const std::string &event, const Config &config) { - sockaddr_un address; + sockaddr_un address{}; address.sun_family = AF_UNIX; - const std::string &uds_path = config.uds_path(); + const auto &uds_path = config.uds_path(); + if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); YagpStat::report_error(); return false; } strcpy(address.sun_path, uds_path.c_str()); - bool success = true; - auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd != -1) { - if (fcntl(sockfd, F_SETFL, O_NONBLOCK) != -1) { - if (connect(sockfd, (sockaddr *)&address, sizeof(address)) != -1) { - auto data_size = req.ByteSize(); - auto total_size = data_size + sizeof(uint32_t); - uint8_t *buf = (uint8_t *)ya_gpdb::palloc(total_size); - uint32_t *size_payload = (uint32_t *)buf; - *size_payload = data_size; - req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); - int64_t sent = 0, sent_total = 0; - do { - sent = send(sockfd, buf + sent_total, total_size - sent_total, - MSG_DONTWAIT); - sent_total += sent; - } while ( - sent > 0 && size_t(sent_total) != total_size && - // the line below is a small throttling hack: - // if a message does not fit a single packet, we take a nap - // before sending the next one. - // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); - if (sent < 0) { - log_tracing_failure(req, event); - success = false; - YagpStat::report_bad_send(total_size); - } else { - YagpStat::report_send(total_size); - } - ya_gpdb::pfree(buf); - } else { - // log the error and go on - log_tracing_failure(req, event); - success = false; - YagpStat::report_bad_connection(); - } - } else { - // That's a very important error that should never happen, so make it - // visible to an end-user and admins. - ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %m"))); - success = false; - YagpStat::report_error(); - } - close(sockfd); - } else { - // log the error and go on + + const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) { log_tracing_failure(req, event); - success = false; YagpStat::report_error(); + return false; } - return success; -} \ No newline at end of file + + // Close socket automatically on error path. + struct SockGuard { + int fd; + ~SockGuard() { close(fd); } + } sock_guard{sockfd}; + + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %m"))); + YagpStat::report_error(); + return false; + } + + if (connect(sockfd, reinterpret_cast(&address), + sizeof(address)) == -1) { + log_tracing_failure(req, event); + YagpStat::report_bad_connection(); + return false; + } + + const auto data_size = req.ByteSize(); + const auto total_size = data_size + sizeof(uint32_t); + auto *buf = static_cast(ya_gpdb::palloc(total_size)); + // Free buf automatically on error path. + struct BufGuard { + void *p; + ~BufGuard() { ya_gpdb::pfree(p); } + } buf_guard{buf}; + + *reinterpret_cast(buf) = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + + int64_t sent = 0, sent_total = 0; + do { + sent = + send(sockfd, buf + sent_total, total_size - sent_total, MSG_DONTWAIT); + if (sent > 0) + sent_total += sent; + } while (sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + + if (sent < 0) { + log_tracing_failure(req, event); + YagpStat::report_bad_send(total_size); + return false; + } + + YagpStat::report_send(total_size); + return true; +} diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 8cf74641c29..602a2470805 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -11,6 +11,12 @@ extern "C" { #include "cdb/ml_ipc.h" #include "tcop/utility.h" #include "stat_statements_parser/pg_stat_statements_ya_parser.h" + +#include +#include +#include +#include +#include } #undef typeid @@ -52,6 +58,13 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); +#define TEST_MAX_CONNECTIONS 4 +#define TEST_RCV_BUF_SIZE 8192 +#define TEST_POLL_TIMEOUT_MS 200 + +static int test_server_fd = -1; +static char *test_sock_path = NULL; + static EventSender *sender = nullptr; static inline EventSender *get_sender() { @@ -226,8 +239,9 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, } get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_DONE, - (void *)query_desc, true /* utility */, (ErrorData *)NULL); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_DONE, (void *)query_desc, true /* utility */, + (ErrorData *)NULL); pfree(query_desc); } @@ -242,8 +256,9 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, MemoryContextSwitchTo(oldctx); get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, METRICS_QUERY_ERROR, - (void *)query_desc, true /* utility */, edata); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_ERROR, (void *)query_desc, true /* utility */, + edata); pfree(query_desc); ReThrowError(edata); @@ -294,4 +309,77 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); +} + +void test_uds_stop_server() { + if (test_server_fd >= 0) { + close(test_server_fd); + test_server_fd = -1; + } + if (test_sock_path) { + unlink(test_sock_path); + pfree(test_sock_path); + test_sock_path = NULL; + } +} + +void test_uds_start_server(const char *path) { + struct sockaddr_un addr = {.sun_family = AF_UNIX}; + + if (strlen(path) >= sizeof(addr.sun_path)) + ereport(ERROR, (errmsg("path too long"))); + + test_uds_stop_server(); + + strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); + test_sock_path = MemoryContextStrdup(TopMemoryContext, path); + unlink(path); + + if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || + bind(test_server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0 || + listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) { + test_uds_stop_server(); + ereport(ERROR, (errmsg("socket setup failed: %m"))); + } +} + +int64 test_uds_receive(int timeout_ms) { + char buf[TEST_RCV_BUF_SIZE]; + int rc; + struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; + int64 total = 0; + + if (test_server_fd < 0) + ereport(ERROR, (errmsg("server not started"))); + + for (;;) { + CHECK_FOR_INTERRUPTS(); + rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); + if (rc > 0) + break; + if (rc < 0 && errno != EINTR) + ereport(ERROR, (errmsg("poll: %m"))); + timeout_ms -= TEST_POLL_TIMEOUT_MS; + if (timeout_ms <= 0) + return total; + } + + if (pfd.revents & POLLIN) { + int client = accept(test_server_fd, NULL, NULL); + ssize_t n; + + if (client < 0) + ereport(ERROR, (errmsg("accept: %m"))); + + while ((n = recv(client, buf, sizeof(buf), 0)) != 0) { + if (n > 0) + total += n; + else if (errno != EINTR) + break; + } + + close(client); + } + + return total; } \ No newline at end of file diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index cfabf39485e..236c6eb9d79 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -12,6 +12,10 @@ extern Datum yagp_functions_get(FunctionCallInfo fcinfo); extern void init_log(); extern void truncate_log(); +extern void test_uds_start_server(const char *path); +extern int64_t test_uds_receive(int timeout_ms); +extern void test_uds_stop_server(); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index 27fd0e04b26..f7863a38921 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -14,16 +14,18 @@ PG_FUNCTION_INFO_V1(yagp_stat_messages); PG_FUNCTION_INFO_V1(yagp_init_log); PG_FUNCTION_INFO_V1(yagp_truncate_log); +PG_FUNCTION_INFO_V1(yagp_test_uds_start_server); +PG_FUNCTION_INFO_V1(yagp_test_uds_receive); +PG_FUNCTION_INFO_V1(yagp_test_uds_stop_server); + void _PG_init(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_init(); - } } void _PG_fini(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) { + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_deinit(); - } } Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { @@ -65,3 +67,57 @@ Datum yagp_truncate_log(PG_FUNCTION_ARGS) { funcctx = SRF_PERCALL_SETUP(); SRF_RETURN_DONE(funcctx); } + +Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + test_uds_start_server(path); + pfree(path); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); +} + +Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + int64 *result; + + if (SRF_IS_FIRSTCALL()) { + MemoryContext oldcontext; + + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + result = (int64 *)palloc(sizeof(int64)); + funcctx->user_fctx = result; + funcctx->max_calls = 1; + MemoryContextSwitchTo(oldcontext); + + int timeout_ms = PG_GETARG_INT32(0); + *result = test_uds_receive(timeout_ms); + } + + funcctx = SRF_PERCALL_SETUP(); + + if (funcctx->call_cntr < funcctx->max_calls) { + result = (int64 *)funcctx->user_fctx; + SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); + } + + SRF_RETURN_DONE(funcctx); +} + +Datum yagp_test_uds_stop_server(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + test_uds_stop_server(); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); +} diff --git a/yagp_hooks_collector--1.1.sql b/yagp_hooks_collector--1.1.sql index e0e94b51493..83bfb553638 100644 --- a/yagp_hooks_collector--1.1.sql +++ b/yagp_hooks_collector--1.1.sql @@ -93,3 +93,18 @@ BEGIN PERFORM yagpcc.__truncate_log_on_segments(); END; $$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION yagpcc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'yagp_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'yagp_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION yagpcc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'yagp_test_uds_stop_server' +LANGUAGE C EXECUTE ON MASTER; From 1a646958ee87f4ab9d20da1a7891d6252476f351 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 21 Jan 2026 12:53:47 +0000 Subject: [PATCH 096/167] [yagp_hooks_collector] Fix locale-dependent normalization crash Make gen_normquery() and gen_normplan() noexcept. Wide-character conversion can fail for locales that cannot handle the input charset. --- expected/yagp_locale.out | 23 ++++++++++++++++++++ gpcontrib/yagp_hooks_collector/Makefile | 2 +- sql/yagp_locale.sql | 29 +++++++++++++++++++++++++ src/ProtoUtils.cpp | 19 ++++++++++------ src/memory/gpdbwrappers.cpp | 11 ++++------ src/memory/gpdbwrappers.h | 4 ++-- 6 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 expected/yagp_locale.out create mode 100644 sql/yagp_locale.sql diff --git a/expected/yagp_locale.out b/expected/yagp_locale.out new file mode 100644 index 00000000000..6689b6a4ed3 --- /dev/null +++ b/expected/yagp_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS yagp_test_locale; +-- end_ignore +CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c yagp_test_locale +CREATE EXTENSION yagp_hooks_collector; +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.enable TO TRUE; +CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +DROP TABLE yagp_hi_안녕세계; +DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index eb6541b7687..d145ae46dbe 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -1,7 +1,7 @@ MODULE_big = yagp_hooks_collector EXTENSION = yagp_hooks_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds +REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds yagp_locale PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) diff --git a/sql/yagp_locale.sql b/sql/yagp_locale.sql new file mode 100644 index 00000000000..65d867d1680 --- /dev/null +++ b/sql/yagp_locale.sql @@ -0,0 +1,29 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. + +-- start_ignore +DROP DATABASE IF EXISTS yagp_test_locale; +-- end_ignore + +CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c yagp_test_locale + +CREATE EXTENSION yagp_hooks_collector; + +SET yagpcc.ignored_users_list TO ''; +SET yagpcc.enable_utility TO TRUE; +SET yagpcc.enable TO TRUE; + +CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; + +RESET yagpcc.enable; +RESET yagpcc.enable_utility; +RESET yagpcc.ignored_users_list; +DROP TABLE yagp_hi_안녕세계; +DROP EXTENSION yagp_hooks_collector; diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index 8ebbe19e289..f9119ca4b14 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -96,13 +96,15 @@ void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); - *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, config.max_plan_size()); - qi->set_plan_id( - hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + if (norm_plan) { + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( + norm_plan->data, norm_plan->len, config.max_plan_size()); + qi->set_plan_id( + hash_any((unsigned char *)norm_plan->data, norm_plan->len)); + ya_gpdb::pfree(norm_plan->data); + } qi->set_query_id(query_desc->plannedstmt->queryId); ya_gpdb::pfree(es.str->data); - ya_gpdb::pfree(norm_plan->data); } ya_gpdb::mem_ctx_switch_to(oldcxt); } @@ -116,8 +118,11 @@ void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, query_desc->sourceText, strlen(query_desc->sourceText), config.max_text_size()); char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); - *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), config.max_text_size()); + if (norm_query) { + *qi->mutable_template_query_text() = trim_str_shrink_utf8( + norm_query, strlen(norm_query), config.max_text_size()); + ya_gpdb::pfree(norm_query); + } } } diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 763e32e539c..8cc483a39de 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -204,15 +204,12 @@ void ya_gpdb::instr_end_loop(Instrumentation *instr) { wrap_throw(::InstrEndLoop, instr); } -char *ya_gpdb::gen_normquery(const char *query) { - return wrap_throw(::gen_normquery, query); +char *ya_gpdb::gen_normquery(const char *query) noexcept { + return wrap_noexcept(::gen_normquery, query); } -StringInfo ya_gpdb::gen_normplan(const char *exec_plan) { - if (!exec_plan) - throw std::runtime_error("Invalid execution plan string"); - - return wrap_throw(::gen_normplan, exec_plan); +StringInfo ya_gpdb::gen_normplan(const char *exec_plan) noexcept { + return wrap_noexcept(::gen_normplan, exec_plan); } char *ya_gpdb::get_rg_name_for_id(Oid group_id) { diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index 920fc1ae6e7..e080ef5cdd4 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -38,8 +38,8 @@ HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime); void instr_end_loop(Instrumentation *instr); -char *gen_normquery(const char *query); -StringInfo gen_normplan(const char *executionPlan); +char *gen_normquery(const char *query) noexcept; +StringInfo gen_normplan(const char *executionPlan) noexcept; char *get_rg_name_for_id(Oid group_id); void insert_log(const yagpcc::SetQueryReq &req, bool utility); From 9a6ee4fb2b5a486687507ceda66cb9b10ec67c41 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 21 Jan 2026 13:52:56 +0000 Subject: [PATCH 097/167] [yagp_hooks_collector] Add Apache license headers and enable -Werror --- gpcontrib/yagp_hooks_collector/Makefile | 2 +- pom.xml | 6 ++++ src/Config.cpp | 27 +++++++++++++++ src/Config.h | 27 +++++++++++++++ src/EventSender.cpp | 27 +++++++++++++++ src/EventSender.h | 27 +++++++++++++++ src/PgUtils.cpp | 27 +++++++++++++++ src/PgUtils.h | 27 +++++++++++++++ src/ProcStats.cpp | 27 +++++++++++++++ src/ProcStats.h | 27 +++++++++++++++ src/ProtoUtils.cpp | 27 +++++++++++++++ src/ProtoUtils.h | 27 +++++++++++++++ src/UDSConnector.cpp | 29 +++++++++++++++- src/UDSConnector.h | 27 +++++++++++++++ src/YagpStat.cpp | 27 +++++++++++++++ src/YagpStat.h | 27 +++++++++++++++ src/hook_wrappers.cpp | 33 +++++++++++++++++-- src/hook_wrappers.h | 27 +++++++++++++++ src/log/LogOps.cpp | 27 +++++++++++++++ src/log/LogOps.h | 27 +++++++++++++++ src/log/LogSchema.cpp | 27 +++++++++++++++ src/log/LogSchema.h | 27 +++++++++++++++ src/memory/gpdbwrappers.cpp | 27 +++++++++++++++ src/memory/gpdbwrappers.h | 27 +++++++++++++++ src/stat_statements_parser/README.md | 1 + .../pg_stat_statements_ya_parser.c | 27 +++++++++++++++ .../pg_stat_statements_ya_parser.h | 27 +++++++++++++++ src/yagp_hooks_collector.c | 27 +++++++++++++++ 28 files changed, 688 insertions(+), 4 deletions(-) create mode 100644 src/stat_statements_parser/README.md diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/yagp_hooks_collector/Makefile index d145ae46dbe..49825c55f35 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/yagp_hooks_collector/Makefile @@ -10,7 +10,7 @@ C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) -override CXXFLAGS = -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ +override CXXFLAGS = -Werror -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD diff --git a/pom.xml b/pom.xml index 4713480a76c..45e62756b11 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,12 @@ code or new licensing patterns. gpcontrib/gp_exttable_fdw/gp_exttable_fdw.control gpcontrib/diskquota/** + gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control + gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto + gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto + gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto + gpcontrib/yagp_hooks_collector/.clang-format + gpcontrib/yagp_hooks_collector/Makefile getversion .git-blame-ignore-revs diff --git a/src/Config.cpp b/src/Config.cpp index 2c2032ebb03..62c16e91d1f 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * Config.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/Config.cpp + * + *------------------------------------------------------------------------- + */ + #include "Config.h" #include "memory/gpdbwrappers.h" #include diff --git a/src/Config.h b/src/Config.h index aa6b5bdc0ba..01ae5ea328e 100644 --- a/src/Config.h +++ b/src/Config.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * Config.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/Config.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/EventSender.cpp b/src/EventSender.cpp index 853a0c43fb9..f1cc0cc6ea1 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * EventSender.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/EventSender.cpp + * + *------------------------------------------------------------------------- + */ + #include "UDSConnector.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" diff --git a/src/EventSender.h b/src/EventSender.h index e9acb04422b..ef7dcb0bf8c 100644 --- a/src/EventSender.h +++ b/src/EventSender.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * EventSender.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/EventSender.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/PgUtils.cpp b/src/PgUtils.cpp index 7e53abdabbf..ed4bf4d7e64 100644 --- a/src/PgUtils.cpp +++ b/src/PgUtils.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PgUtils.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/PgUtils.cpp + * + *------------------------------------------------------------------------- + */ + #include "PgUtils.h" #include "Config.h" #include "memory/gpdbwrappers.h" diff --git a/src/PgUtils.h b/src/PgUtils.h index e9715ce10f4..5113fadbff2 100644 --- a/src/PgUtils.h +++ b/src/PgUtils.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PgUtils.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/PgUtils.h + * + *------------------------------------------------------------------------- + */ + extern "C" { #include "postgres.h" #include "commands/explain.h" diff --git a/src/ProcStats.cpp b/src/ProcStats.cpp index 5c09fa0bce4..72a12e8ca00 100644 --- a/src/ProcStats.cpp +++ b/src/ProcStats.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProcStats.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProcStats.cpp + * + *------------------------------------------------------------------------- + */ + #include "ProcStats.h" #include "yagpcc_metrics.pb.h" #include diff --git a/src/ProcStats.h b/src/ProcStats.h index 30a90a60519..7629edd0aea 100644 --- a/src/ProcStats.h +++ b/src/ProcStats.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProcStats.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProcStats.h + * + *------------------------------------------------------------------------- + */ + #pragma once namespace yagpcc { diff --git a/src/ProtoUtils.cpp b/src/ProtoUtils.cpp index f9119ca4b14..b449ae20900 100644 --- a/src/ProtoUtils.cpp +++ b/src/ProtoUtils.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProtoUtils.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp + * + *------------------------------------------------------------------------- + */ + #include "ProtoUtils.h" #include "PgUtils.h" #include "ProcStats.h" diff --git a/src/ProtoUtils.h b/src/ProtoUtils.h index 37b7e4a8a29..c954545494f 100644 --- a/src/ProtoUtils.h +++ b/src/ProtoUtils.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * ProtoUtils.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/ProtoUtils.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include "protos/yagpcc_set_service.pb.h" diff --git a/src/UDSConnector.cpp b/src/UDSConnector.cpp index ea118fca783..d13a82a5ca9 100644 --- a/src/UDSConnector.cpp +++ b/src/UDSConnector.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * UDSConnector.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp + * + *------------------------------------------------------------------------- + */ + #include "UDSConnector.h" #include "Config.h" #include "YagpStat.h" @@ -67,7 +94,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, return false; } - const auto data_size = req.ByteSize(); + const auto data_size = req.ByteSizeLong(); const auto total_size = data_size + sizeof(uint32_t); auto *buf = static_cast(ya_gpdb::palloc(total_size)); // Free buf automatically on error path. diff --git a/src/UDSConnector.h b/src/UDSConnector.h index 9483407159d..be5ab1ef413 100644 --- a/src/UDSConnector.h +++ b/src/UDSConnector.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * UDSConnector.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/UDSConnector.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include "protos/yagpcc_set_service.pb.h" diff --git a/src/YagpStat.cpp b/src/YagpStat.cpp index 879cde85212..3a760b6ea97 100644 --- a/src/YagpStat.cpp +++ b/src/YagpStat.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * YagpStat.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/YagpStat.cpp + * + *------------------------------------------------------------------------- + */ + #include "YagpStat.h" #include diff --git a/src/YagpStat.h b/src/YagpStat.h index 110b1fdcbb1..57fc90cd4d1 100644 --- a/src/YagpStat.h +++ b/src/YagpStat.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * YagpStat.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/YagpStat.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/hook_wrappers.cpp b/src/hook_wrappers.cpp index 602a2470805..cb4970d60d9 100644 --- a/src/hook_wrappers.cpp +++ b/src/hook_wrappers.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * hook_wrappers.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp + * + *------------------------------------------------------------------------- + */ + #define typeid __typeid extern "C" { #include "postgres.h" @@ -46,8 +73,10 @@ static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, static void ya_ExecutorFinish_hook(QueryDesc *query_desc); static void ya_ExecutorEnd_hook(QueryDesc *query_desc); static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +#ifdef IC_TEARDOWN_HOOK static void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); +#endif #ifdef ANALYZE_STATS_COLLECT_HOOK static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); #endif @@ -195,14 +224,14 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } } +#ifdef IC_TEARDOWN_HOOK void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { cpp_call(get_sender(), &EventSender::ic_metrics_collect); -#ifdef IC_TEARDOWN_HOOK if (previous_ic_teardown_hook) { (*previous_ic_teardown_hook)(transportStates, hasErrors); } -#endif } +#endif #ifdef ANALYZE_STATS_COLLECT_HOOK void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { diff --git a/src/hook_wrappers.h b/src/hook_wrappers.h index 236c6eb9d79..443406a5259 100644 --- a/src/hook_wrappers.h +++ b/src/hook_wrappers.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * hook_wrappers.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/hook_wrappers.h + * + *------------------------------------------------------------------------- + */ + #pragma once #ifdef __cplusplus diff --git a/src/log/LogOps.cpp b/src/log/LogOps.cpp index 56bdf1dca62..e8c927ece84 100644 --- a/src/log/LogOps.cpp +++ b/src/log/LogOps.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogOps.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp + * + *------------------------------------------------------------------------- + */ + #include "protos/yagpcc_set_service.pb.h" #include "LogOps.h" diff --git a/src/log/LogOps.h b/src/log/LogOps.h index bad03d09a8f..1fc30c21030 100644 --- a/src/log/LogOps.h +++ b/src/log/LogOps.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogOps.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogOps.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/log/LogSchema.cpp b/src/log/LogSchema.cpp index 2fadcc46599..a391b1a2209 100644 --- a/src/log/LogSchema.cpp +++ b/src/log/LogSchema.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogSchema.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp + * + *------------------------------------------------------------------------- + */ + #include "google/protobuf/reflection.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/timestamp.pb.h" diff --git a/src/log/LogSchema.h b/src/log/LogSchema.h index f713c1e9b0e..f78acec7ce9 100644 --- a/src/log/LogSchema.h +++ b/src/log/LogSchema.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * LogSchema.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/log/LogSchema.h + * + *------------------------------------------------------------------------- + */ + #pragma once #include diff --git a/src/memory/gpdbwrappers.cpp b/src/memory/gpdbwrappers.cpp index 8cc483a39de..22083e8bdaf 100644 --- a/src/memory/gpdbwrappers.cpp +++ b/src/memory/gpdbwrappers.cpp @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gpdbwrappers.cpp + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp + * + *------------------------------------------------------------------------- + */ + #include "gpdbwrappers.h" #include "log/LogOps.h" diff --git a/src/memory/gpdbwrappers.h b/src/memory/gpdbwrappers.h index e080ef5cdd4..fe9b3ba0487 100644 --- a/src/memory/gpdbwrappers.h +++ b/src/memory/gpdbwrappers.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gpdbwrappers.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h + * + *------------------------------------------------------------------------- + */ + #pragma once extern "C" { diff --git a/src/stat_statements_parser/README.md b/src/stat_statements_parser/README.md new file mode 100644 index 00000000000..291e31a3099 --- /dev/null +++ b/src/stat_statements_parser/README.md @@ -0,0 +1 @@ +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 54c8b2cf59f..7404208055f 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_stat_statements_ya_parser.c + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * + *------------------------------------------------------------------------- + */ + // NOTE: this file is just a bunch of code borrowed from pg_stat_statements for PG 9.4 // and from our own inhouse implementation of pg_stat_statements for managed PG diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/src/stat_statements_parser/pg_stat_statements_ya_parser.h index b08e8533992..96c6a776dba 100644 --- a/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_stat_statements_ya_parser.h + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * + *------------------------------------------------------------------------- + */ + #pragma once #ifdef __cplusplus diff --git a/src/yagp_hooks_collector.c b/src/yagp_hooks_collector.c index f7863a38921..271bceee178 100644 --- a/src/yagp_hooks_collector.c +++ b/src/yagp_hooks_collector.c @@ -1,3 +1,30 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * yagp_hooks_collector.c + * + * IDENTIFICATION + * gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c + * + *------------------------------------------------------------------------- + */ + #include "postgres.h" #include "cdb/cdbvars.h" #include "funcapi.h" From 4f976780d690e09ac656450923b66bfa138f8132 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Tue, 10 Feb 2026 10:41:58 +0300 Subject: [PATCH 098/167] [yagp_hooks_collector] Fix null ErrorData dereference on segments Guard against NULL ErrorData in set_qi_error_message(). For some query types ErrorData can be NULL despite an error occurring. --- src/EventSender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventSender.cpp b/src/EventSender.cpp index f1cc0cc6ea1..6993814ffbf 100644 --- a/src/EventSender.cpp +++ b/src/EventSender.cpp @@ -290,7 +290,7 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { bool error_flushed = elog_message() == NULL; - if (error_flushed && edata->message == NULL) { + if (error_flushed && (edata == NULL || edata->message == NULL)) { ereport(WARNING, (errmsg("YAGPCC missing error message"))); ereport(DEBUG3, (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); From 13e7b603b727eb1fa898ee0b15674bf7a7ed9b3f Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Mon, 29 Jun 2026 15:58:40 +0300 Subject: [PATCH 099/167] Fix conflicts with yagp_hooks_collector --- src/stat_statements_parser/README.MD | 1 - src/stat_statements_parser/README.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 src/stat_statements_parser/README.MD delete mode 100644 src/stat_statements_parser/README.md diff --git a/src/stat_statements_parser/README.MD b/src/stat_statements_parser/README.MD deleted file mode 100644 index 291e31a3099..00000000000 --- a/src/stat_statements_parser/README.MD +++ /dev/null @@ -1 +0,0 @@ -This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/src/stat_statements_parser/README.md b/src/stat_statements_parser/README.md deleted file mode 100644 index 291e31a3099..00000000000 --- a/src/stat_statements_parser/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. From 147ab8672223684669dfbff309f2883bd9dc48af Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Wed, 18 Mar 2026 15:17:52 +0000 Subject: [PATCH 100/167] [yagp_hooks_collector] Move to gpcontrib directory --- .../yagp_hooks_collector/.clang-format | 0 gpcontrib/yagp_hooks_collector/README.md | 28 +++++++++++ .../expected}/yagp_cursors.out | 0 .../expected}/yagp_dist.out | 0 .../expected}/yagp_guc_cache.out | 0 .../expected}/yagp_locale.out | 0 .../expected}/yagp_select.out | 0 .../expected}/yagp_uds.out | 0 .../expected}/yagp_utf8_trim.out | 0 .../expected}/yagp_utility.out | 0 .../yagp_hooks_collector/metric.md | 0 .../protos}/yagpcc_metrics.proto | 0 .../protos}/yagpcc_plan.proto | 0 .../protos}/yagpcc_set_service.proto | 0 .../sql}/yagp_cursors.sql | 0 .../yagp_hooks_collector/sql}/yagp_dist.sql | 0 .../sql}/yagp_guc_cache.sql | 0 .../yagp_hooks_collector/sql}/yagp_locale.sql | 0 .../yagp_hooks_collector/sql}/yagp_select.sql | 0 .../yagp_hooks_collector/sql}/yagp_uds.sql | 0 .../sql}/yagp_utf8_trim.sql | 0 .../sql}/yagp_utility.sql | 0 .../yagp_hooks_collector/src}/Config.cpp | 0 .../yagp_hooks_collector/src}/Config.h | 0 .../yagp_hooks_collector/src}/EventSender.cpp | 0 .../yagp_hooks_collector/src}/EventSender.h | 0 .../yagp_hooks_collector/src}/PgUtils.cpp | 0 .../yagp_hooks_collector/src}/PgUtils.h | 0 .../yagp_hooks_collector/src}/ProcStats.cpp | 0 .../yagp_hooks_collector/src}/ProcStats.h | 0 .../yagp_hooks_collector/src}/ProtoUtils.cpp | 0 .../yagp_hooks_collector/src}/ProtoUtils.h | 0 .../src}/UDSConnector.cpp | 0 .../yagp_hooks_collector/src}/UDSConnector.h | 0 .../yagp_hooks_collector/src}/YagpStat.cpp | 0 .../yagp_hooks_collector/src}/YagpStat.h | 0 .../src}/hook_wrappers.cpp | 0 .../yagp_hooks_collector/src}/hook_wrappers.h | 0 .../yagp_hooks_collector/src}/log/LogOps.cpp | 0 .../yagp_hooks_collector/src}/log/LogOps.h | 0 .../src}/log/LogSchema.cpp | 0 .../yagp_hooks_collector/src}/log/LogSchema.h | 0 .../src}/memory/gpdbwrappers.cpp | 0 .../src}/memory/gpdbwrappers.h | 0 .../src/stat_statements_parser/README.md | 47 +++++++++++++++++++ .../pg_stat_statements_ya_parser.c | 0 .../pg_stat_statements_ya_parser.h | 0 .../src}/yagp_hooks_collector.c | 0 .../yagp_hooks_collector--1.0--1.1.sql | 0 .../yagp_hooks_collector--1.0.sql | 0 .../yagp_hooks_collector--1.1.sql | 0 .../yagp_hooks_collector.control | 0 52 files changed, 75 insertions(+) rename .clang-format => gpcontrib/yagp_hooks_collector/.clang-format (100%) create mode 100644 gpcontrib/yagp_hooks_collector/README.md rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_cursors.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_dist.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_guc_cache.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_locale.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_select.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_uds.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_utf8_trim.out (100%) rename {expected => gpcontrib/yagp_hooks_collector/expected}/yagp_utility.out (100%) rename metric.md => gpcontrib/yagp_hooks_collector/metric.md (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_metrics.proto (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_plan.proto (100%) rename {protos => gpcontrib/yagp_hooks_collector/protos}/yagpcc_set_service.proto (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_cursors.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_dist.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_guc_cache.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_locale.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_select.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_uds.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_utf8_trim.sql (100%) rename {sql => gpcontrib/yagp_hooks_collector/sql}/yagp_utility.sql (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/Config.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/Config.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/EventSender.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/EventSender.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/PgUtils.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/PgUtils.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProcStats.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProcStats.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProtoUtils.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/ProtoUtils.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/UDSConnector.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/UDSConnector.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/YagpStat.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/YagpStat.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/hook_wrappers.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/hook_wrappers.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogOps.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogOps.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogSchema.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/log/LogSchema.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/memory/gpdbwrappers.cpp (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/memory/gpdbwrappers.h (100%) create mode 100644 gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md rename {src => gpcontrib/yagp_hooks_collector/src}/stat_statements_parser/pg_stat_statements_ya_parser.c (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/stat_statements_parser/pg_stat_statements_ya_parser.h (100%) rename {src => gpcontrib/yagp_hooks_collector/src}/yagp_hooks_collector.c (100%) rename yagp_hooks_collector--1.0--1.1.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql (100%) rename yagp_hooks_collector--1.0.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql (100%) rename yagp_hooks_collector--1.1.sql => gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql (100%) rename yagp_hooks_collector.control => gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control (100%) diff --git a/.clang-format b/gpcontrib/yagp_hooks_collector/.clang-format similarity index 100% rename from .clang-format rename to gpcontrib/yagp_hooks_collector/.clang-format diff --git a/gpcontrib/yagp_hooks_collector/README.md b/gpcontrib/yagp_hooks_collector/README.md new file mode 100644 index 00000000000..9f465a190cb --- /dev/null +++ b/gpcontrib/yagp_hooks_collector/README.md @@ -0,0 +1,28 @@ +## YAGP Hooks Collector + +An extension for collecting greenplum query execution metrics and reporting them to an external agent. + +### Collected Statistics + +#### 1. Query Lifecycle +- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. +- **GUC:** `yagpcc.enable`. + +#### 2. `EXPLAIN` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. +- **GUC:** `yagpcc.enable`. + +#### 3. `EXPLAIN ANALYZE` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. +- **GUCs:** `yagpcc.enable`, `yagpcc.min_analyze_time`, `yagpcc.enable_cdbstats`(ANALYZE), `yagpcc.enable_analyze`(BUFFERS, TIMING, VERBOSE). + +#### 4. Other Metrics +- **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics. +- **GUC:** `yagpcc.enable`. + +### General Configuration +- **Nested Queries:** When `yagpcc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. +- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `yagpcc.uds_path`. +- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `yagpcc.ignored_users_list`. +- **Trimming plans:** Query texts and execution plans are trimmed based on `yagpcc.max_text_size` and `yagpcc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. +- **Analyze collection:** Analyze is sent if execution time exceeds `yagpcc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `yagpcc.enable_analyze` is true. diff --git a/expected/yagp_cursors.out b/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out similarity index 100% rename from expected/yagp_cursors.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out diff --git a/expected/yagp_dist.out b/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out similarity index 100% rename from expected/yagp_dist.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_dist.out diff --git a/expected/yagp_guc_cache.out b/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out similarity index 100% rename from expected/yagp_guc_cache.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out diff --git a/expected/yagp_locale.out b/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out similarity index 100% rename from expected/yagp_locale.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_locale.out diff --git a/expected/yagp_select.out b/gpcontrib/yagp_hooks_collector/expected/yagp_select.out similarity index 100% rename from expected/yagp_select.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_select.out diff --git a/expected/yagp_uds.out b/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out similarity index 100% rename from expected/yagp_uds.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_uds.out diff --git a/expected/yagp_utf8_trim.out b/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out similarity index 100% rename from expected/yagp_utf8_trim.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out diff --git a/expected/yagp_utility.out b/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out similarity index 100% rename from expected/yagp_utility.out rename to gpcontrib/yagp_hooks_collector/expected/yagp_utility.out diff --git a/metric.md b/gpcontrib/yagp_hooks_collector/metric.md similarity index 100% rename from metric.md rename to gpcontrib/yagp_hooks_collector/metric.md diff --git a/protos/yagpcc_metrics.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto similarity index 100% rename from protos/yagpcc_metrics.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto diff --git a/protos/yagpcc_plan.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto similarity index 100% rename from protos/yagpcc_plan.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto diff --git a/protos/yagpcc_set_service.proto b/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto similarity index 100% rename from protos/yagpcc_set_service.proto rename to gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto diff --git a/sql/yagp_cursors.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql similarity index 100% rename from sql/yagp_cursors.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql diff --git a/sql/yagp_dist.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql similarity index 100% rename from sql/yagp_dist.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql diff --git a/sql/yagp_guc_cache.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql similarity index 100% rename from sql/yagp_guc_cache.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql diff --git a/sql/yagp_locale.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql similarity index 100% rename from sql/yagp_locale.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql diff --git a/sql/yagp_select.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql similarity index 100% rename from sql/yagp_select.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_select.sql diff --git a/sql/yagp_uds.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql similarity index 100% rename from sql/yagp_uds.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql diff --git a/sql/yagp_utf8_trim.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql similarity index 100% rename from sql/yagp_utf8_trim.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql diff --git a/sql/yagp_utility.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql similarity index 100% rename from sql/yagp_utility.sql rename to gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql diff --git a/src/Config.cpp b/gpcontrib/yagp_hooks_collector/src/Config.cpp similarity index 100% rename from src/Config.cpp rename to gpcontrib/yagp_hooks_collector/src/Config.cpp diff --git a/src/Config.h b/gpcontrib/yagp_hooks_collector/src/Config.h similarity index 100% rename from src/Config.h rename to gpcontrib/yagp_hooks_collector/src/Config.h diff --git a/src/EventSender.cpp b/gpcontrib/yagp_hooks_collector/src/EventSender.cpp similarity index 100% rename from src/EventSender.cpp rename to gpcontrib/yagp_hooks_collector/src/EventSender.cpp diff --git a/src/EventSender.h b/gpcontrib/yagp_hooks_collector/src/EventSender.h similarity index 100% rename from src/EventSender.h rename to gpcontrib/yagp_hooks_collector/src/EventSender.h diff --git a/src/PgUtils.cpp b/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp similarity index 100% rename from src/PgUtils.cpp rename to gpcontrib/yagp_hooks_collector/src/PgUtils.cpp diff --git a/src/PgUtils.h b/gpcontrib/yagp_hooks_collector/src/PgUtils.h similarity index 100% rename from src/PgUtils.h rename to gpcontrib/yagp_hooks_collector/src/PgUtils.h diff --git a/src/ProcStats.cpp b/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp similarity index 100% rename from src/ProcStats.cpp rename to gpcontrib/yagp_hooks_collector/src/ProcStats.cpp diff --git a/src/ProcStats.h b/gpcontrib/yagp_hooks_collector/src/ProcStats.h similarity index 100% rename from src/ProcStats.h rename to gpcontrib/yagp_hooks_collector/src/ProcStats.h diff --git a/src/ProtoUtils.cpp b/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp similarity index 100% rename from src/ProtoUtils.cpp rename to gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp diff --git a/src/ProtoUtils.h b/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h similarity index 100% rename from src/ProtoUtils.h rename to gpcontrib/yagp_hooks_collector/src/ProtoUtils.h diff --git a/src/UDSConnector.cpp b/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp similarity index 100% rename from src/UDSConnector.cpp rename to gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp diff --git a/src/UDSConnector.h b/gpcontrib/yagp_hooks_collector/src/UDSConnector.h similarity index 100% rename from src/UDSConnector.h rename to gpcontrib/yagp_hooks_collector/src/UDSConnector.h diff --git a/src/YagpStat.cpp b/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp similarity index 100% rename from src/YagpStat.cpp rename to gpcontrib/yagp_hooks_collector/src/YagpStat.cpp diff --git a/src/YagpStat.h b/gpcontrib/yagp_hooks_collector/src/YagpStat.h similarity index 100% rename from src/YagpStat.h rename to gpcontrib/yagp_hooks_collector/src/YagpStat.h diff --git a/src/hook_wrappers.cpp b/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp similarity index 100% rename from src/hook_wrappers.cpp rename to gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp diff --git a/src/hook_wrappers.h b/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h similarity index 100% rename from src/hook_wrappers.h rename to gpcontrib/yagp_hooks_collector/src/hook_wrappers.h diff --git a/src/log/LogOps.cpp b/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp similarity index 100% rename from src/log/LogOps.cpp rename to gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp diff --git a/src/log/LogOps.h b/gpcontrib/yagp_hooks_collector/src/log/LogOps.h similarity index 100% rename from src/log/LogOps.h rename to gpcontrib/yagp_hooks_collector/src/log/LogOps.h diff --git a/src/log/LogSchema.cpp b/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp similarity index 100% rename from src/log/LogSchema.cpp rename to gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp diff --git a/src/log/LogSchema.h b/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h similarity index 100% rename from src/log/LogSchema.h rename to gpcontrib/yagp_hooks_collector/src/log/LogSchema.h diff --git a/src/memory/gpdbwrappers.cpp b/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp similarity index 100% rename from src/memory/gpdbwrappers.cpp rename to gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp diff --git a/src/memory/gpdbwrappers.h b/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h similarity index 100% rename from src/memory/gpdbwrappers.h rename to gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md new file mode 100644 index 00000000000..8c2d5c6868e --- /dev/null +++ b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md @@ -0,0 +1,47 @@ + + +## GP Stats Collector + +An extension for collecting query execution metrics and reporting them to an external agent. + +### Collected Statistics + +#### 1. Query Lifecycle +- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. +- **GUC:** `gpsc.enable`. + +#### 2. `EXPLAIN` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. +- **GUC:** `gpsc.enable`. + +#### 3. `EXPLAIN ANALYZE` data +- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. +- **GUCs:** `gpsc.enable`, `gpsc.min_analyze_time`, `gpsc.enable_cdbstats`(ANALYZE), `gpsc.enable_analyze`(BUFFERS, TIMING, VERBOSE). + +#### 4. Other Metrics +- **What:** Captures Instrument, System, Network, Interconnect, Spill metrics. +- **GUC:** `gpsc.enable`. + +### General Configuration +- **Nested Queries:** When `gpsc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. +- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `gpsc.uds_path`. +- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`. +- **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. +- **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true. diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c similarity index 100% rename from src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c diff --git a/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h similarity index 100% rename from src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h diff --git a/src/yagp_hooks_collector.c b/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c similarity index 100% rename from src/yagp_hooks_collector.c rename to gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c diff --git a/yagp_hooks_collector--1.0--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql similarity index 100% rename from yagp_hooks_collector--1.0--1.1.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql diff --git a/yagp_hooks_collector--1.0.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql similarity index 100% rename from yagp_hooks_collector--1.0.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql diff --git a/yagp_hooks_collector--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql similarity index 100% rename from yagp_hooks_collector--1.1.sql rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql diff --git a/yagp_hooks_collector.control b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control similarity index 100% rename from yagp_hooks_collector.control rename to gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control From 7c8b12a9138ba395115ba18ae6996789e81a5ee2 Mon Sep 17 00:00:00 2001 From: NJrslv <108277031+NJrslv@users.noreply.github.com> Date: Wed, 25 Mar 2026 18:37:39 +0300 Subject: [PATCH 101/167] [gp_stats_collector] Rename yagp_hooks_collector to gp_stats_collector Rename extension, shared library, SQL objects, GUC prefix, test files, and all internal identifiers. Restore accidentally deleted files. Clean up stray gmon.out. --- .github/workflows/build-cloudberry-rocky8.yml | 32 ++- .github/workflows/build-cloudberry.yml | 16 +- .github/workflows/build-deb-cloudberry.yml | 32 ++- .gitignore | 5 - Makefile | 2 + configure | 3 +- configure.ac | 19 +- .../.clang-format | 0 gpcontrib/gp_stats_collector/.gitignore | 5 + .../Makefile | 16 +- .../README.md | 0 .../expected/gpsc_cursors.out} | 72 ++--- .../expected/gpsc_dist.out} | 56 ++-- .../expected/gpsc_guc_cache.out} | 32 +-- .../expected/gpsc_locale.out | 23 ++ .../expected/gpsc_select.out} | 56 ++-- .../gp_stats_collector/expected/gpsc_uds.out | 42 +++ .../expected/gpsc_utf8_trim.out} | 36 +-- .../expected/gpsc_utility.out} | 172 ++++++------ .../gp_stats_collector--1.0--1.1.sql | 113 ++++++++ .../gp_stats_collector--1.0.sql | 55 ++++ .../gp_stats_collector--1.1.sql | 110 ++++++++ .../gp_stats_collector.control | 5 + .../metric.md | 27 +- .../protos/gpsc_metrics.proto} | 4 +- .../protos/gpsc_plan.proto} | 4 +- .../protos/gpsc_set_service.proto} | 8 +- .../results/gpsc_cursors.out | 163 ++++++++++++ .../gp_stats_collector/results/gpsc_dist.out | 175 ++++++++++++ .../results/gpsc_guc_cache.out | 61 +++++ .../results/gpsc_locale.out | 23 ++ .../results/gpsc_select.out | 136 ++++++++++ .../gp_stats_collector/results/gpsc_uds.out | 42 +++ .../results/gpsc_utf8_trim.out | 68 +++++ .../results/gpsc_utility.out | 248 ++++++++++++++++++ .../gp_stats_collector/sql/gpsc_cursors.sql | 85 ++++++ .../sql/gpsc_dist.sql} | 58 ++-- .../sql/gpsc_guc_cache.sql} | 32 +-- .../gp_stats_collector/sql/gpsc_locale.sql | 29 ++ .../gp_stats_collector/sql/gpsc_select.sql | 69 +++++ gpcontrib/gp_stats_collector/sql/gpsc_uds.sql | 31 +++ .../sql/gpsc_utf8_trim.sql} | 36 +-- .../gp_stats_collector/sql/gpsc_utility.sql | 135 ++++++++++ .../src/Config.cpp | 46 ++-- .../src/Config.h | 2 +- .../src/EventSender.cpp | 78 +++--- .../src/EventSender.h | 32 +-- .../src/GpscStat.cpp} | 36 +-- .../src/GpscStat.h} | 6 +- .../src/PgUtils.cpp | 20 +- .../src/PgUtils.h | 2 +- .../src/ProcStats.cpp | 12 +- .../src/ProcStats.h | 6 +- .../src/ProtoUtils.cpp | 60 ++--- .../src/ProtoUtils.h | 26 +- .../src/UDSConnector.cpp | 24 +- .../src/UDSConnector.h | 6 +- .../src/gp_stats_collector.c} | 36 +-- .../src/hook_wrappers.cpp | 72 ++--- .../src/hook_wrappers.h | 6 +- .../src/log/LogOps.cpp | 16 +- .../src/log/LogOps.h | 10 +- .../src/log/LogSchema.cpp | 10 +- .../src/log/LogSchema.h | 8 +- .../src/memory/gpdbwrappers.cpp | 42 +-- .../src/memory/gpdbwrappers.h | 12 +- .../src/stat_statements_parser/README.md | 20 ++ .../pg_stat_statements_ya_parser.c | 2 +- .../pg_stat_statements_ya_parser.h | 2 +- gpcontrib/yagp_hooks_collector/README.md | 28 -- .../expected/yagp_locale.out | 23 -- .../expected/yagp_uds.out | 42 --- .../yagp_hooks_collector/sql/yagp_cursors.sql | 85 ------ .../yagp_hooks_collector/sql/yagp_locale.sql | 29 -- .../yagp_hooks_collector/sql/yagp_select.sql | 69 ----- .../yagp_hooks_collector/sql/yagp_uds.sql | 31 --- .../yagp_hooks_collector/sql/yagp_utility.sql | 135 ---------- .../yagp_hooks_collector--1.0--1.1.sql | 113 -------- .../yagp_hooks_collector--1.0.sql | 55 ---- .../yagp_hooks_collector--1.1.sql | 110 -------- .../yagp_hooks_collector.control | 5 - pom.xml | 16 +- src/backend/commands/portalcmds.c | 2 +- src/backend/tcop/pquery.c | 4 +- src/include/executor/execdesc.h | 8 +- 85 files changed, 2339 insertions(+), 1344 deletions(-) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/.clang-format (100%) create mode 100644 gpcontrib/gp_stats_collector/.gitignore rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/Makefile (66%) rename gpcontrib/{yagp_hooks_collector/src/stat_statements_parser => gp_stats_collector}/README.md (100%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_cursors.out => gp_stats_collector/expected/gpsc_cursors.out} (73%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_dist.out => gp_stats_collector/expected/gpsc_dist.out} (81%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_guc_cache.out => gp_stats_collector/expected/gpsc_guc_cache.out} (64%) create mode 100644 gpcontrib/gp_stats_collector/expected/gpsc_locale.out rename gpcontrib/{yagp_hooks_collector/expected/yagp_select.out => gp_stats_collector/expected/gpsc_select.out} (67%) create mode 100644 gpcontrib/gp_stats_collector/expected/gpsc_uds.out rename gpcontrib/{yagp_hooks_collector/expected/yagp_utf8_trim.out => gp_stats_collector/expected/gpsc_utf8_trim.out} (65%) rename gpcontrib/{yagp_hooks_collector/expected/yagp_utility.out => gp_stats_collector/expected/gpsc_utility.out} (57%) create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector.control rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/metric.md (94%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_metrics.proto => gp_stats_collector/protos/gpsc_metrics.proto} (97%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_plan.proto => gp_stats_collector/protos/gpsc_plan.proto} (98%) rename gpcontrib/{yagp_hooks_collector/protos/yagpcc_set_service.proto => gp_stats_collector/protos/gpsc_set_service.proto} (86%) create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_cursors.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_dist.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_locale.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_select.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_uds.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out create mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utility.out create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql rename gpcontrib/{yagp_hooks_collector/sql/yagp_dist.sql => gp_stats_collector/sql/gpsc_dist.sql} (53%) rename gpcontrib/{yagp_hooks_collector/sql/yagp_guc_cache.sql => gp_stats_collector/sql/gpsc_guc_cache.sql} (58%) create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_locale.sql create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_select.sql create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_uds.sql rename gpcontrib/{yagp_hooks_collector/sql/yagp_utf8_trim.sql => gp_stats_collector/sql/gpsc_utf8_trim.sql} (58%) create mode 100644 gpcontrib/gp_stats_collector/sql/gpsc_utility.sql rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/Config.cpp (79%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/Config.h (97%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/EventSender.cpp (86%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/EventSender.h (84%) rename gpcontrib/{yagp_hooks_collector/src/YagpStat.cpp => gp_stats_collector/src/GpscStat.cpp} (78%) rename gpcontrib/{yagp_hooks_collector/src/YagpStat.h => gp_stats_collector/src/GpscStat.h} (94%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/PgUtils.cpp (83%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/PgUtils.h (96%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProcStats.cpp (92%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProcStats.h (89%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProtoUtils.cpp (85%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/ProtoUtils.h (65%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/UDSConnector.cpp (87%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/UDSConnector.h (88%) rename gpcontrib/{yagp_hooks_collector/src/yagp_hooks_collector.c => gp_stats_collector/src/gp_stats_collector.c} (79%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/hook_wrappers.cpp (84%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/hook_wrappers.h (89%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogOps.cpp (91%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogOps.h (83%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogSchema.cpp (94%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/log/LogSchema.h (98%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/memory/gpdbwrappers.cpp (81%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/memory/gpdbwrappers.h (92%) create mode 100644 gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/stat_statements_parser/pg_stat_statements_ya_parser.c (99%) rename gpcontrib/{yagp_hooks_collector => gp_stats_collector}/src/stat_statements_parser/pg_stat_statements_ya_parser.h (93%) delete mode 100644 gpcontrib/yagp_hooks_collector/README.md delete mode 100644 gpcontrib/yagp_hooks_collector/expected/yagp_locale.out delete mode 100644 gpcontrib/yagp_hooks_collector/expected/yagp_uds.out delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_select.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql delete mode 100644 gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql delete mode 100644 gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index cf966722264..f009539d37d 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -321,6 +321,10 @@ jobs: "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, {"test":"ic-fixme", "make_configs":["src/test/regress:installcheck-fixme"], "enable_core_check":false @@ -541,10 +545,11 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1401,6 +1406,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: /usr/local/cloudberry-db shell: bash {0} run: | set -o pipefail @@ -1424,6 +1430,30 @@ jobs: # 2. Follow the same pattern as optimizer # 3. Update matrix entries to include the new setting + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + # Set PostgreSQL options if defined PG_OPTS="" if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 6b956954700..94b38529d21 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -271,9 +271,9 @@ jobs: }, "enable_core_check":false }, - {"test":"gpcontrib-yagp-hooks-collector", - "make_configs":["gpcontrib/yagp_hooks_collector:installcheck"], - "extension":"yagp_hooks_collector" + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] @@ -541,7 +541,7 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-yagp-hooks-collector + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -1443,17 +1443,17 @@ jobs: # Create extension if required if [[ "${{ matrix.extension != '' }}" == "true" ]]; then case "${{ matrix.extension }}" in - yagp_hooks_collector) + gp_stats_collector) if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ - gpconfig -c shared_preload_libraries -v 'yagp_hooks_collector' && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ gpstop -ra && \ - echo 'CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ SHOW shared_preload_libraries; \ TABLE pg_extension;' | \ psql postgres" then - echo "Error creating yagp_hooks_collector extension" + echo "Error creating gp_stats_collector extension" exit 1 fi ;; diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index fee69b073f7..592ef2eaf69 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -253,6 +253,10 @@ jobs: "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck"] }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, {"test":"ic-cbdb-parallel", "make_configs":["src/test/regress:installcheck-cbdb-parallel"] } @@ -449,13 +453,14 @@ jobs: shell: bash env: SRC_DIR: ${{ github.workspace }} + CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail export BUILD_DESTINATION=${SRC_DIR}/debian/build chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi @@ -1342,6 +1347,7 @@ jobs: if: success() && needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: ${{ github.workspace }}/debian/build shell: bash {0} run: | set -o pipefail @@ -1366,6 +1372,30 @@ jobs: # 3. Update matrix entries to include the new setting + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + # Set PostgreSQL options if defined PG_OPTS="" if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then diff --git a/.gitignore b/.gitignore index 29b40ee096c..7f5110d5c8e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,8 +74,3 @@ lib*.pc /tmp_install/ /.cache/ /install/ -*.o -*.so -src/protos/ -.vscode -compile_commands.json diff --git a/Makefile b/Makefile index 15c5dabb70e..e9ab3fbf2d4 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,14 @@ # to build Postgres with a different make, we have this make file # that, as a service, will look for a GNU make and invoke it, or show # an error message if none could be found. + # If the user were using GNU make now, this file would not get used # because GNU make uses a make file named "GNUmakefile" in preference # to "Makefile" if it exists. PostgreSQL is shipped with a # "GNUmakefile". If the user hasn't run the configure script yet, the # GNUmakefile won't exist yet, so we catch that case as well. + # AIX make defaults to building *every* target of the first rule. Start with # a single-target, empty rule to make the other targets non-default. all: diff --git a/configure b/configure index e2d0232ead9..bcbe741a543 100755 --- a/configure +++ b/configure @@ -723,7 +723,7 @@ with_libcurl with_rt with_zstd with_yezzey -with_yagp_hooks_collector +with_gp_stats_collector with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -947,7 +947,6 @@ with_zstd with_diskquota with_gp_stats_collector with_yezzey -with_yagp_hooks_collector with_rt with_libcurl with_apr_config diff --git a/configure.ac b/configure.ac index a2a69b069f9..3ca5585f1b6 100644 --- a/configure.ac +++ b/configure.ac @@ -1369,11 +1369,22 @@ AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) # -# yagp_hooks_collector +# gp_stats_collector # -PGAC_ARG_BOOL(with, yagp_hooks_collector, no, - [build with YAGP hooks collector extension]) -AC_SUBST(with_yagp_hooks_collector) +PGAC_ARG_BOOL(with, gp_stats_collector, no, + [build with stats collector extension]) +AC_SUBST(with_gp_stats_collector) + +if test "$with_gp_stats_collector" = yes; then + PKG_CHECK_MODULES([PROTOBUF], [protobuf >= 3.0.0], + [], + [AC_MSG_ERROR([protobuf >= 3.0.0 is required for gp_stats_collector])] + ) + AC_PATH_PROG([PROTOC], [protoc], [no]) + if test "$PROTOC" = no; then + AC_MSG_ERROR([protoc is required for gp_stats_collector but was not found in PATH]) + fi +fi if test "$with_zstd" = yes; then dnl zstd_errors.h was renamed from error_public.h in v1.4.0 diff --git a/gpcontrib/yagp_hooks_collector/.clang-format b/gpcontrib/gp_stats_collector/.clang-format similarity index 100% rename from gpcontrib/yagp_hooks_collector/.clang-format rename to gpcontrib/gp_stats_collector/.clang-format diff --git a/gpcontrib/gp_stats_collector/.gitignore b/gpcontrib/gp_stats_collector/.gitignore new file mode 100644 index 00000000000..e8dfe855dad --- /dev/null +++ b/gpcontrib/gp_stats_collector/.gitignore @@ -0,0 +1,5 @@ +*.o +*.so +src/protos/ +.vscode +compile_commands.json diff --git a/gpcontrib/yagp_hooks_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile similarity index 66% rename from gpcontrib/yagp_hooks_collector/Makefile rename to gpcontrib/gp_stats_collector/Makefile index 49825c55f35..c8f7b3c30fe 100644 --- a/gpcontrib/yagp_hooks_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -1,9 +1,9 @@ -MODULE_big = yagp_hooks_collector -EXTENSION = yagp_hooks_collector +MODULE_big = gp_stats_collector +EXTENSION = gp_stats_collector DATA = $(wildcard *--*.sql) -REGRESS = yagp_cursors yagp_dist yagp_select yagp_utf8_trim yagp_utility yagp_guc_cache yagp_uds yagp_locale +REGRESS = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_uds gpsc_locale -PROTO_BASES = yagpcc_plan yagpcc_metrics yagpcc_set_service +PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) @@ -24,7 +24,7 @@ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) else -subdir = gpcontrib/yagp_hooks_collector +subdir = gpcontrib/gp_stats_collector top_builddir = ../.. include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk @@ -32,10 +32,8 @@ endif src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto @mkdir -p src/protos - sed -i 's/optional //g' $^ - sed -i 's|cloud/mdb/yagpcc/api/proto/common/|protos/|g' $^ protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ mv src/protos/$*.pb.cc src/protos/$*.pb.cpp -$(CPP_OBJS): src/protos/yagpcc_metrics.pb.h src/protos/yagpcc_plan.pb.h src/protos/yagpcc_set_service.pb.h -src/protos/yagpcc_set_service.pb.o: src/protos/yagpcc_metrics.pb.h +$(CPP_OBJS): src/protos/gpsc_metrics.pb.h src/protos/gpsc_plan.pb.h src/protos/gpsc_set_service.pb.h +src/protos/gpsc_set_service.pb.o: src/protos/gpsc_metrics.pb.h diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md b/gpcontrib/gp_stats_collector/README.md similarity index 100% rename from gpcontrib/yagp_hooks_collector/src/stat_statements_parser/README.md rename to gpcontrib/gp_stats_collector/README.md diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out b/gpcontrib/gp_stats_collector/expected/gpsc_cursors.out similarity index 73% rename from gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out rename to gpcontrib/gp_stats_collector/expected/gpsc_cursors.out index df12e3e1b66..282d9ac49e1 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_cursors.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_cursors.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,18 +12,18 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; -- DECLARE -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_0 CURSOR FOR SELECT 0; CLOSE cursor_stats_0; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -34,25 +34,25 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- DECLARE WITH HOLD -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; CLOSE cursor_stats_1; DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; CLOSE cursor_stats_2; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -67,24 +67,24 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (14 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- ROLLBACK -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_3 CURSOR FOR SELECT 1; CLOSE cursor_stats_3; DECLARE cursor_stats_4 CURSOR FOR SELECT 1; ROLLBACK; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -97,17 +97,17 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE -1 | ROLLBACK; | QUERY_STATUS_SUBMIT -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (12 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- FETCH -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; @@ -126,8 +126,8 @@ FETCH 1 IN cursor_stats_6; CLOSE cursor_stats_5; CLOSE cursor_stats_6; COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------------+--------------------- -1 | BEGIN; | QUERY_STATUS_SUBMIT @@ -146,18 +146,18 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE -1 | COMMIT; | QUERY_STATUS_SUBMIT -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (18 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out b/gpcontrib/gp_stats_collector/expected/gpsc_dist.out similarity index 81% rename from gpcontrib/yagp_hooks_collector/expected/yagp_dist.out rename to gpcontrib/gp_stats_collector/expected/gpsc_dist.out index 3b1e3504923..92e8678767b 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_dist.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_dist.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,14 +12,14 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Hash distributed table CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); INSERT INTO test_hash_dist SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SET optimizer_enable_direct_dispatch TO TRUE; -- Direct dispatch is used here, only one segment is scanned. select * from test_hash_dist where id = 1; @@ -29,9 +29,9 @@ select * from test_hash_dist where id = 1; (1 row) RESET optimizer_enable_direct_dispatch; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Should see 8 rows. -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------------------+--------------------- -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT @@ -44,12 +44,12 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag 1 | | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; -- Scan all segments. select * from test_hash_dist; id @@ -58,8 +58,8 @@ select * from test_hash_dist; (1 row) DROP TABLE test_hash_dist; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------+--------------------- -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT @@ -80,7 +80,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | | QUERY_STATUS_DONE (16 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) @@ -93,7 +93,7 @@ END; $$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; INSERT INTO test_replicated SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_replicated, force_segments(); count ------- @@ -102,8 +102,8 @@ SELECT COUNT(*) FROM test_replicated, force_segments(); DROP TABLE test_replicated; DROP FUNCTION force_segments(); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------------------+--------------------- -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT @@ -124,7 +124,7 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | | QUERY_STATUS_DONE (16 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) @@ -134,18 +134,18 @@ SET allow_system_table_mods = ON; CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_partial_dist; count ------- 100 (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; DROP TABLE test_partial_dist; RESET allow_system_table_mods; -- Should see 12 rows. -SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; query_text | query_status -----------------------------------------+--------------------- SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT @@ -162,14 +162,14 @@ SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_statu | QUERY_STATUS_DONE (12 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out b/gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out similarity index 64% rename from gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out rename to gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out index 3085cfa42e1..11a420839db 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_guc_cache.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_guc_cache.out @@ -8,23 +8,23 @@ -- have its DONE event rejected, creating orphaned SUBMIT entries. -- This is due to query being actually executed between SUBMIT and DONE. -- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; -SELECT yagpcc.truncate_log(); +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); -- end_ignore CREATE OR REPLACE FUNCTION print_last_query(query text) RETURNS TABLE(query_status text) AS $$ SELECT query_status - FROM yagpcc.log + FROM gpsc.log WHERE segid = -1 AND query_text = query ORDER BY ccnt DESC $$ LANGUAGE sql; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.logging_mode TO 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; -- SET below disables utility logging and DONE must still be logged. -SET yagpcc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); query_status --------------------- QUERY_STATUS_SUBMIT @@ -33,14 +33,14 @@ SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); -- SELECT below adds current user to ignore list and DONE must still be logged. -- start_ignore -SELECT set_config('yagpcc.ignored_users_list', current_user, false); +SELECT set_config('gpsc.ignored_users_list', current_user, false); set_config ------------ gpadmin (1 row) -- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); query_status --------------------- QUERY_STATUS_SUBMIT @@ -50,8 +50,8 @@ SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', (4 rows) DROP FUNCTION print_last_query(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -RESET yagpcc.logging_mode; +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/expected/gpsc_locale.out b/gpcontrib/gp_stats_collector/expected/gpsc_locale.out new file mode 100644 index 00000000000..a01fe0648b9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/expected/gpsc_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale +CREATE EXTENSION gp_stats_collector; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_select.out b/gpcontrib/gp_stats_collector/expected/gpsc_select.out similarity index 67% rename from gpcontrib/yagp_hooks_collector/expected/yagp_select.out rename to gpcontrib/gp_stats_collector/expected/gpsc_select.out index af08f2d1def..3008c8f6d55 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_select.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_select.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,12 +12,12 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Basic SELECT tests -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT 1; ?column? ---------- @@ -30,8 +30,8 @@ SELECT COUNT(*) FROM generate_series(1,10); 10 (1 row) -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------------------+--------------------- -1 | SELECT 1; | QUERY_STATUS_SUBMIT @@ -44,13 +44,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Transaction test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; SELECT 1; ?column? @@ -59,8 +59,8 @@ SELECT 1; (1 row) COMMIT; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+------------+--------------------- -1 | SELECT 1; | QUERY_STATUS_SUBMIT @@ -69,13 +69,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | SELECT 1; | QUERY_STATUS_DONE (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- CTE test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; WITH t AS (VALUES (1), (2)) SELECT * FROM t; column1 @@ -84,8 +84,8 @@ SELECT * FROM t; 2 (2 rows) -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-----------------------------+--------------------- -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT @@ -98,13 +98,13 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag | SELECT * FROM t; | (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Prepared statement test -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; PREPARE test_stmt AS SELECT 1; EXECUTE test_stmt; ?column? @@ -113,8 +113,8 @@ EXECUTE test_stmt; (1 row) DEALLOCATE test_stmt; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+--------------------------------+--------------------- -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT @@ -123,14 +123,14 @@ SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yag -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE (4 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/expected/gpsc_uds.out b/gpcontrib/gp_stats_collector/expected/gpsc_uds.out new file mode 100644 index 00000000000..e8bca79e669 --- /dev/null +++ b/gpcontrib/gp_stats_collector/expected/gpsc_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +\set UDS_PATH '/tmp/gpsc_test.sock' +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out b/gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out similarity index 65% rename from gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out rename to gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out index 9de126dd882..db3949f3152 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_utf8_trim.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_utf8_trim.out @@ -1,24 +1,24 @@ -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) RETURNS TEXT AS $$ SELECT query_text - FROM yagpcc.log + FROM gpsc.log WHERE query_text LIKE '%' || marker || '%' ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; -- Test 1: 1 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test1*/ 'HelloWorld'; ?column? ------------ HelloWorld (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; correct_length ---------------- @@ -26,15 +26,15 @@ SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; (1 row) -- Test 2: 2 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test2*/ 'РУССКИЙЯЗЫК'; ?column? ------------- РУССКИЙЯЗЫК (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Character 'Р' has two bytes and cut in the middle => not included. SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; correct_length @@ -43,15 +43,15 @@ SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; (1 row) -- Test 3: 4 byte chars -SET yagpcc.max_text_size to 21; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; SELECT /*test3*/ '😀'; ?column? ---------- 😀 (1 row) -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Emoji has 4 bytes and cut before the last byte => not included. SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; correct_length @@ -61,8 +61,8 @@ SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; -- Cleanup DROP FUNCTION get_marked_query(TEXT); -RESET yagpcc.max_text_size; -RESET yagpcc.logging_mode; -RESET yagpcc.enable; -RESET yagpcc.ignored_users_list; -DROP EXTENSION yagp_hooks_collector; +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out b/gpcontrib/gp_stats_collector/expected/gpsc_utility.out similarity index 57% rename from gpcontrib/yagp_hooks_collector/expected/yagp_utility.out rename to gpcontrib/gp_stats_collector/expected/gpsc_utility.out index 7df1d2816eb..e8e28614370 100644 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_utility.out +++ b/gpcontrib/gp_stats_collector/expected/gpsc_utility.out @@ -1,5 +1,5 @@ -CREATE EXTENSION yagp_hooks_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -12,19 +12,19 @@ BEGIN END; END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE test_table (a int, b text); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. CREATE INDEX test_idx ON test_table(a); ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; DROP TABLE test_table; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------------------+--------------------- -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT @@ -35,24 +35,24 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT -1 | DROP TABLE test_table; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Partitioning -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE pt_test (a int, b int) DISTRIBUTED BY (a) PARTITION BY RANGE (a) (START (0) END (100) EVERY (50)); DROP TABLE pt_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------+--------------------- -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT @@ -65,23 +65,23 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util | (START (0) END (100) EVERY (50)); | -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Views and Functions -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE VIEW test_view AS SELECT 1 AS a; CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; DROP VIEW test_view; DROP FUNCTION test_func(int); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+------------------------------------------------------------------------------------+--------------------- -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT @@ -92,17 +92,17 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | DROP VIEW test_view; | QUERY_STATUS_DONE -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (10 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Transaction Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; BEGIN; SAVEPOINT sp1; ROLLBACK TO sp1; @@ -112,37 +112,37 @@ SAVEPOINT sp2; ABORT; BEGIN; ROLLBACK; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (18 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- DML Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE dml_test (a int, b text); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. @@ -150,33 +150,33 @@ INSERT INTO dml_test VALUES (1, 'test'); UPDATE dml_test SET b = 'updated' WHERE a = 1; DELETE FROM dml_test WHERE a = 1; DROP TABLE dml_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+----------------------------------------+--------------------- -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- COPY Operations -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; CREATE TABLE copy_test (a int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. COPY (SELECT 1) TO STDOUT; 1 DROP TABLE copy_test; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+---------------------------------+--------------------- -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT @@ -185,23 +185,23 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- Prepared Statements and error during execute -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; PREPARE test_prep(int) AS SELECT $1/0 AS value; EXECUTE test_prep(0::int); ERROR: division by zero DEALLOCATE test_prep; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; segid | query_text | query_status -------+-------------------------------------------------+--------------------- -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT @@ -210,39 +210,39 @@ SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND util -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (8 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -- GUC Settings -SET yagpcc.logging_mode to 'TBL'; -SET yagpcc.report_nested_queries TO FALSE; -RESET yagpcc.report_nested_queries; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------------------+--------------------- - -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT - -1 | SET yagpcc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE - -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.report_nested_queries; | QUERY_STATUS_DONE - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET yagpcc.logging_mode; | QUERY_STATUS_DONE +SET gpsc.logging_mode to 'TBL'; +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------+--------------------- + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE (6 rows) -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT gpsc.truncate_log() IS NOT NULL AS t; t --- (0 rows) -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql new file mode 100644 index 00000000000..4e0157117e9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql @@ -0,0 +1,113 @@ +/* gp_stats_collector--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_stats_collector UPDATE TO '1.1'" to load this file. \quit + +CREATE SCHEMA gpsc; + +-- Unlink existing objects from extension. +ALTER EXTENSION gp_stats_collector DROP VIEW gpsc_stat_messages; +ALTER EXTENSION gp_stats_collector DROP FUNCTION gpsc_stat_messages_reset(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_f_on_segments(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_f_on_master(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_reset_f_on_segments(); +ALTER EXTENSION gp_stats_collector DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); + +-- Now drop the objects. +DROP VIEW gpsc_stat_messages; +DROP FUNCTION gpsc_stat_messages_reset(); +DROP FUNCTION __gpsc_stat_messages_f_on_segments(); +DROP FUNCTION __gpsc_stat_messages_f_on_master(); +DROP FUNCTION __gpsc_stat_messages_reset_f_on_segments(); +DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); + +-- Recreate functions and view in new schema. +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +-- Create new objects. +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments + ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql new file mode 100644 index 00000000000..ec902b02e02 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql @@ -0,0 +1,55 @@ +/* gp_stats_collector--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE FUNCTION __gpsc_stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc_stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT __gpsc_stat_messages_reset_f_on_master(); + SELECT __gpsc_stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION __gpsc_stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc_stat_messages AS + SELECT C.* + FROM __gpsc_stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM __gpsc_stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql new file mode 100644 index 00000000000..6e24207e913 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql @@ -0,0 +1,110 @@ +/* gp_stats_collector--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE SCHEMA gpsc; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION gpsc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON MASTER; + +CREATE FUNCTION gpsc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' +LANGUAGE C EXECUTE ON MASTER; diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector.control b/gpcontrib/gp_stats_collector/gp_stats_collector.control new file mode 100644 index 00000000000..4aea2bd49b8 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector.control @@ -0,0 +1,5 @@ +# gp_stats_collector extension +comment = 'Intercept query and plan execution hooks and report them to Cloudberry monitor agents' +default_version = '1.1' +module_pathname = '$libdir/gp_stats_collector' +superuser = true diff --git a/gpcontrib/yagp_hooks_collector/metric.md b/gpcontrib/gp_stats_collector/metric.md similarity index 94% rename from gpcontrib/yagp_hooks_collector/metric.md rename to gpcontrib/gp_stats_collector/metric.md index 5df56877edb..6f168d8cd98 100644 --- a/gpcontrib/yagp_hooks_collector/metric.md +++ b/gpcontrib/gp_stats_collector/metric.md @@ -1,4 +1,23 @@ -## YAGP Hooks Collector Metrics + + +## GP Stats Collector Metrics ### States A Postgres process goes through 4 executor functions to execute a query: @@ -7,7 +26,7 @@ A Postgres process goes through 4 executor functions to execute a query: 3) `ExecutorFinish()` - cleanup. 4) `ExecutorEnd()` - cleanup. -yagp-hooks-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: +gp-stats-collector sends messages with 4 states, from _Dispatcher_ and/or _Execute_ processes: `submit`, `start`, `end`, `done`, in this order: ``` submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done ``` @@ -67,8 +86,8 @@ submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end - | `temp_blks_written` | uint64 | E, D | ABS | + | Node | + | + | blocks | Temp file blocks written | | `blk_read_time` | double | E, D | ABS | + | Node | + | + | seconds | Time reading data blocks | | `blk_write_time` | double | E, D | ABS | + | Node | + | + | seconds | Time writing data blocks | -| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (YAGPCC-specific) | -| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (YAGPCC-specific) | +| `inherited_calls` | uint64 | E, D | ABS | - | Node | + | + | count | Nested query count (GPSC-specific) | +| `inherited_time` | double | E, D | ABS | - | Node | + | + | seconds | Nested query time (GPSC-specific) | | **NetworkStat (sent)** | | | | | | | | | | | `sent.total_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes sent, including headers | | `sent.tuple_bytes` | uint32 | D | ABS | - | Node | + | + | bytes | Bytes of pure tuple-data sent | diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto similarity index 97% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index 91ac0c4941a..a9e26471839 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -1,8 +1,6 @@ syntax = "proto3"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCM"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; +package gpsc; enum QueryStatus { QUERY_STATUS_UNSPECIFIED = 0; diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto similarity index 98% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_plan.proto index 962fab4bbdd..5a7269edd20 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto @@ -1,8 +1,6 @@ syntax = "proto3"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCP"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/common;greenplum"; +package gpsc; message MetricPlan { GpdbNodeType type = 1; diff --git a/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto similarity index 86% rename from gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto rename to gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto index 0b9e34df49d..4cd795424ab 100644 --- a/gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto @@ -2,12 +2,10 @@ syntax = "proto3"; import "google/protobuf/timestamp.proto"; -import "protos/yagpcc_metrics.proto"; -import "protos/yagpcc_plan.proto"; +import "protos/gpsc_metrics.proto"; +import "protos/gpsc_plan.proto"; -package yagpcc; -option java_outer_classname = "SegmentYAGPCCAS"; -option go_package = "a.yandex-team.ru/cloud/mdb/yagpcc/api/proto/agent_segment;greenplum"; +package gpsc; service SetQueryInfo { rpc SetMetricPlanNode (SetPlanNodeReq) returns (MetricResponse) {} diff --git a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out new file mode 100644 index 00000000000..282d9ac49e1 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out @@ -0,0 +1,163 @@ +CREATE EXTENSION gp_stats_collector; +CREATE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +-- DECLARE +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- DECLARE WITH HOLD +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(14 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- ROLLBACK +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(12 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- FETCH +SET gpsc.logging_mode to 'TBL'; +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; + ?column? +---------- + 2 +(1 row) + +FETCH 1 IN cursor_stats_6; + ?column? +---------- + 3 +(1 row) + +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT + -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT + -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(18 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_dist.out b/gpcontrib/gp_stats_collector/results/gpsc_dist.out new file mode 100644 index 00000000000..92e8678767b --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_dist.out @@ -0,0 +1,175 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; +-- Hash distributed table +CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); +INSERT INTO test_hash_dist SELECT 1; +SET gpsc.logging_mode to 'TBL'; +SET optimizer_enable_direct_dispatch TO TRUE; +-- Direct dispatch is used here, only one segment is scanned. +select * from test_hash_dist where id = 1; + id +---- + 1 +(1 row) + +RESET optimizer_enable_direct_dispatch; +RESET gpsc.logging_mode; +-- Should see 8 rows. +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------------------+--------------------- + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END + -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +SET gpsc.logging_mode to 'TBL'; +-- Scan all segments. +select * from test_hash_dist; + id +---- + 1 +(1 row) + +DROP TABLE test_hash_dist; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------+--------------------- + -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT + -1 | select * from test_hash_dist; | QUERY_STATUS_START + -1 | select * from test_hash_dist; | QUERY_STATUS_END + -1 | select * from test_hash_dist; | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Replicated table +CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ +BEGIN + RETURN NEXT 'seg'; +END; +$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; +CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; +INSERT INTO test_replicated SELECT 1; +SET gpsc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_replicated, force_segments(); + count +------- + 3 +(1 row) + +DROP TABLE test_replicated; +DROP FUNCTION force_segments(); +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------------------+--------------------- + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE + 1 | | QUERY_STATUS_SUBMIT + 1 | | QUERY_STATUS_START + 1 | | QUERY_STATUS_END + 1 | | QUERY_STATUS_DONE + 2 | | QUERY_STATUS_SUBMIT + 2 | | QUERY_STATUS_START + 2 | | QUERY_STATUS_END + 2 | | QUERY_STATUS_DONE + | | QUERY_STATUS_SUBMIT + | | QUERY_STATUS_START + | | QUERY_STATUS_END + | | QUERY_STATUS_DONE +(16 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Partially distributed table (2 numsegments) +SET allow_system_table_mods = ON; +CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); +UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; +INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); +SET gpsc.logging_mode to 'TBL'; +SELECT COUNT(*) FROM test_partial_dist; + count +------- + 100 +(1 row) + +RESET gpsc.logging_mode; +DROP TABLE test_partial_dist; +RESET allow_system_table_mods; +-- Should see 12 rows. +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + query_text | query_status +-----------------------------------------+--------------------- + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END + SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE + | QUERY_STATUS_SUBMIT + | QUERY_STATUS_START + | QUERY_STATUS_END + | QUERY_STATUS_DONE +(12 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out new file mode 100644 index 00000000000..19c4774575d --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out @@ -0,0 +1,61 @@ +-- +-- Test GUC caching for query lifecycle consistency. +-- +-- The extension logs SUBMIT and DONE events for each query. +-- GUC values that control logging (enable_utility, ignored_users_list, ...) +-- must be cached at SUBMIT time to ensure DONE uses the same filtering +-- criteria. Otherwise, a SET command that modifies these GUCs would +-- have its DONE event rejected, creating orphaned SUBMIT entries. +-- This is due to query being actually executed between SUBMIT and DONE. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); + truncate_log +-------------- +(0 rows) + +-- end_ignore +CREATE OR REPLACE FUNCTION print_last_query(query text) +RETURNS TABLE(query_status text) AS $$ + SELECT query_status + FROM gpsc.log + WHERE segid = -1 AND query_text = query + ORDER BY ccnt DESC +$$ LANGUAGE sql; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; +-- SET below disables utility logging and DONE must still be logged. +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_DONE +(2 rows) + +-- SELECT below adds current user to ignore list and DONE must still be logged. +-- start_ignore +SELECT set_config('gpsc.ignored_users_list', current_user, false); + set_config +------------ + gpadmin +(1 row) + +-- end_ignore +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); + query_status +--------------------- + QUERY_STATUS_SUBMIT + QUERY_STATUS_START + QUERY_STATUS_END + QUERY_STATUS_DONE +(4 rows) + +DROP FUNCTION print_last_query(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_locale.out b/gpcontrib/gp_stats_collector/results/gpsc_locale.out new file mode 100644 index 00000000000..a01fe0648b9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_locale.out @@ -0,0 +1,23 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale +CREATE EXTENSION gp_stats_collector; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_select.out b/gpcontrib/gp_stats_collector/results/gpsc_select.out new file mode 100644 index 00000000000..3008c8f6d55 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_select.out @@ -0,0 +1,136 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; +-- Basic SELECT tests +SET gpsc.logging_mode to 'TBL'; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +SELECT COUNT(*) FROM generate_series(1,10); + count +------- + 10 +(1 row) + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Transaction test +SET gpsc.logging_mode to 'TBL'; +BEGIN; +SELECT 1; + ?column? +---------- + 1 +(1 row) + +COMMIT; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- CTE test +SET gpsc.logging_mode to 'TBL'; +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + column1 +--------- + 1 + 2 +(2 rows) + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-----------------------------+--------------------- + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END + | SELECT * FROM t; | + -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE + | SELECT * FROM t; | +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Prepared statement test +SET gpsc.logging_mode to 'TBL'; +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; + ?column? +---------- + 1 +(1 row) + +DEALLOCATE test_stmt; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------------+--------------------- + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END + -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE +(4 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_uds.out b/gpcontrib/gp_stats_collector/results/gpsc_uds.out new file mode 100644 index 00000000000..e8bca79e669 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_uds.out @@ -0,0 +1,42 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +\set UDS_PATH '/tmp/gpsc_test.sock' +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + __test_uds_start_server +------------------------- +(0 rows) + +-- Send +SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + received +---------- + t +(1 row) + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + __test_uds_stop_server +------------------------ +(0 rows) + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out new file mode 100644 index 00000000000..db3949f3152 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out @@ -0,0 +1,68 @@ +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) +RETURNS TEXT AS $$ + SELECT query_text + FROM gpsc.log + WHERE query_text LIKE '%' || marker || '%' + ORDER BY datetime DESC + LIMIT 1 +$$ LANGUAGE sql VOLATILE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +-- Test 1: 1 byte chars +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test1*/ 'HelloWorld'; + ?column? +------------ + HelloWorld +(1 row) + +RESET gpsc.logging_mode; +SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 2: 2 byte chars +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test2*/ 'РУССКИЙЯЗЫК'; + ?column? +------------- + РУССКИЙЯЗЫК +(1 row) + +RESET gpsc.logging_mode; +-- Character 'Р' has two bytes and cut in the middle => not included. +SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Test 3: 4 byte chars +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; +SELECT /*test3*/ '😀'; + ?column? +---------- + 😀 +(1 row) + +RESET gpsc.logging_mode; +-- Emoji has 4 bytes and cut before the last byte => not included. +SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; + correct_length +---------------- + t +(1 row) + +-- Cleanup +DROP FUNCTION get_marked_query(TEXT); +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utility.out b/gpcontrib/gp_stats_collector/results/gpsc_utility.out new file mode 100644 index 00000000000..e8e28614370 --- /dev/null +++ b/gpcontrib/gp_stats_collector/results/gpsc_utility.out @@ -0,0 +1,248 @@ +CREATE EXTENSION gp_stats_collector; +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE test_table (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------------------+--------------------- + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT + -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT + -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE + -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE test_table; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Partitioning +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------+--------------------- + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE + | DISTRIBUTED BY (a) +| + | PARTITION BY RANGE (a) +| + | (START (0) END (100) EVERY (50)); | + -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Views and Functions +SET gpsc.logging_mode to 'TBL'; +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------------------------------------------------+--------------------- + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT + -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT + -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE + -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT + -1 | DROP VIEW test_view; | QUERY_STATUS_DONE + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT + -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(10 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Transaction Operations +SET gpsc.logging_mode to 'TBL'; +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; +BEGIN; +SAVEPOINT sp2; +ABORT; +BEGIN; +ROLLBACK; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+--------------------------+--------------------- + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE + -1 | COMMIT; | QUERY_STATUS_SUBMIT + -1 | COMMIT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_SUBMIT + -1 | ABORT; | QUERY_STATUS_DONE + -1 | BEGIN; | QUERY_STATUS_SUBMIT + -1 | BEGIN; | QUERY_STATUS_DONE + -1 | ROLLBACK; | QUERY_STATUS_SUBMIT + -1 | ROLLBACK; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(18 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- DML Operations +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE dml_test (a int, b text); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+----------------------------------------+--------------------- + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE + -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- COPY Operations +SET gpsc.logging_mode to 'TBL'; +CREATE TABLE copy_test (a int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +COPY (SELECT 1) TO STDOUT; +1 +DROP TABLE copy_test; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+---------------------------------+--------------------- + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT + -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT + -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE + -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT + -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- Prepared Statements and error during execute +SET gpsc.logging_mode to 'TBL'; +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +ERROR: division by zero +DEALLOCATE test_prep; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+-------------------------------------------------+--------------------- + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT + -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT + -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR + -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT + -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(8 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +-- GUC Settings +SET gpsc.logging_mode to 'TBL'; +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; + segid | query_text | query_status +-------+------------------------------------------+--------------------- + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT + -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT + -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE +(6 rows) + +SELECT gpsc.truncate_log() IS NOT NULL AS t; + t +--- +(0 rows) + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql b/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql new file mode 100644 index 00000000000..8361f7b678d --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_cursors.sql @@ -0,0 +1,85 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; + +-- DECLARE +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_0 CURSOR FOR SELECT 0; +CLOSE cursor_stats_0; +COMMIT; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- DECLARE WITH HOLD +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; +CLOSE cursor_stats_1; +DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; +CLOSE cursor_stats_2; +COMMIT; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- ROLLBACK +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_3 CURSOR FOR SELECT 1; +CLOSE cursor_stats_3; +DECLARE cursor_stats_4 CURSOR FOR SELECT 1; +ROLLBACK; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- FETCH +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; +DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; +FETCH 1 IN cursor_stats_5; +FETCH 1 IN cursor_stats_6; +CLOSE cursor_stats_5; +CLOSE cursor_stats_6; +COMMIT; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql b/gpcontrib/gp_stats_collector/sql/gpsc_dist.sql similarity index 53% rename from gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_dist.sql index d5519d0cd96..46b531a70ca 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_dist.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_dist.sql @@ -1,6 +1,6 @@ -CREATE EXTENSION yagp_hooks_collector; +CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION yagp_status_order(status text) +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) RETURNS integer AS $$ BEGIN @@ -14,36 +14,36 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; -- Hash distributed table CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); INSERT INTO test_hash_dist SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SET optimizer_enable_direct_dispatch TO TRUE; -- Direct dispatch is used here, only one segment is scanned. select * from test_hash_dist where id = 1; RESET optimizer_enable_direct_dispatch; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Should see 8 rows. -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; -- Scan all segments. select * from test_hash_dist; DROP TABLE test_hash_dist; -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -- Replicated table CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ @@ -55,14 +55,14 @@ $$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; INSERT INTO test_replicated SELECT 1; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_replicated, force_segments(); DROP TABLE test_replicated; DROP FUNCTION force_segments(); -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; -- Partially distributed table (2 numsegments) SET allow_system_table_mods = ON; @@ -70,19 +70,19 @@ CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.logging_mode to 'TBL'; SELECT COUNT(*) FROM test_partial_dist; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; DROP TABLE test_partial_dist; RESET allow_system_table_mods; -- Should see 12 rows. -SELECT query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; +SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql b/gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql similarity index 58% rename from gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql index 9e6de69d61e..6aff2ad5cf6 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_guc_cache.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_guc_cache.sql @@ -8,36 +8,36 @@ -- have its DONE event rejected, creating orphaned SUBMIT entries. -- This is due to query being actually executed between SUBMIT and DONE. -- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; -SELECT yagpcc.truncate_log(); +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +SELECT gpsc.truncate_log(); -- end_ignore CREATE OR REPLACE FUNCTION print_last_query(query text) RETURNS TABLE(query_status text) AS $$ SELECT query_status - FROM yagpcc.log + FROM gpsc.log WHERE segid = -1 AND query_text = query ORDER BY ccnt DESC $$ LANGUAGE sql; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.logging_mode TO 'TBL'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.logging_mode TO 'TBL'; -- SET below disables utility logging and DONE must still be logged. -SET yagpcc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET yagpcc.enable_utility TO FALSE;'); +SET gpsc.enable_utility TO FALSE; +SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); -- SELECT below adds current user to ignore list and DONE must still be logged. -- start_ignore -SELECT set_config('yagpcc.ignored_users_list', current_user, false); +SELECT set_config('gpsc.ignored_users_list', current_user, false); -- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''yagpcc.ignored_users_list'', current_user, false);'); +SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); DROP FUNCTION print_last_query(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -RESET yagpcc.logging_mode; +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql b/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql new file mode 100644 index 00000000000..6321c93f5ab --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_locale.sql @@ -0,0 +1,29 @@ +-- The extension generates normalized query text and plan using jumbling functions. +-- Those functions may fail when translating to wide character if the current locale +-- cannot handle the character set. This test checks that even when those functions +-- fail, the plan is still generated and executed. This test is partially taken from +-- gp_locale. + +-- start_ignore +DROP DATABASE IF EXISTS gpsc_test_locale; +-- end_ignore + +CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; +\c gpsc_test_locale + +CREATE EXTENSION gp_stats_collector; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable_utility TO TRUE; +SET gpsc.enable TO TRUE; + +CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); +INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); +-- Should not see error here +UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; + +RESET gpsc.enable; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; +DROP TABLE gpsc_hi_안녕세계; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_select.sql b/gpcontrib/gp_stats_collector/sql/gpsc_select.sql new file mode 100644 index 00000000000..673cbee0c10 --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_select.sql @@ -0,0 +1,69 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; + +-- Basic SELECT tests +SET gpsc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Transaction test +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +SELECT 1; +COMMIT; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- CTE test +SET gpsc.logging_mode to 'TBL'; + +WITH t AS (VALUES (1), (2)) +SELECT * FROM t; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Prepared statement test +SET gpsc.logging_mode to 'TBL'; + +PREPARE test_stmt AS SELECT 1; +EXECUTE test_stmt; +DEALLOCATE test_stmt; + +RESET gpsc.logging_mode; +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql b/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql new file mode 100644 index 00000000000..14377b15c8c --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_uds.sql @@ -0,0 +1,31 @@ +-- Test UDS socket +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore + +\set UDS_PATH '/tmp/gpsc_test.sock' + +-- Configure extension to send via UDS +SET gpsc.uds_path TO :'UDS_PATH'; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.logging_mode TO 'UDS'; + +-- Start receiver +SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); + +-- Send +SELECT 1; + +-- Receive +SELECT gpsc.__test_uds_receive() > 0 as received; + +-- Stop receiver +SELECT gpsc.__test_uds_stop_server(); + +-- Cleanup +DROP EXTENSION gp_stats_collector; +RESET gpsc.uds_path; +RESET gpsc.ignored_users_list; +RESET gpsc.enable; +RESET gpsc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql b/gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql similarity index 58% rename from gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql rename to gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql index c3053e4af0c..a3f8a376d55 100644 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_utf8_trim.sql +++ b/gpcontrib/gp_stats_collector/sql/gpsc_utf8_trim.sql @@ -1,45 +1,45 @@ -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) RETURNS TEXT AS $$ SELECT query_text - FROM yagpcc.log + FROM gpsc.log WHERE query_text LIKE '%' || marker || '%' ORDER BY datetime DESC LIMIT 1 $$ LANGUAGE sql VOLATILE; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; -- Test 1: 1 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test1*/ 'HelloWorld'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; -- Test 2: 2 byte chars -SET yagpcc.max_text_size to 19; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 19; +SET gpsc.logging_mode to 'TBL'; SELECT /*test2*/ 'РУССКИЙЯЗЫК'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Character 'Р' has two bytes and cut in the middle => not included. SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; -- Test 3: 4 byte chars -SET yagpcc.max_text_size to 21; -SET yagpcc.logging_mode to 'TBL'; +SET gpsc.max_text_size to 21; +SET gpsc.logging_mode to 'TBL'; SELECT /*test3*/ '😀'; -RESET yagpcc.logging_mode; +RESET gpsc.logging_mode; -- Emoji has 4 bytes and cut before the last byte => not included. SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; -- Cleanup DROP FUNCTION get_marked_query(TEXT); -RESET yagpcc.max_text_size; -RESET yagpcc.logging_mode; -RESET yagpcc.enable; -RESET yagpcc.ignored_users_list; +RESET gpsc.max_text_size; +RESET gpsc.logging_mode; +RESET gpsc.enable; +RESET gpsc.ignored_users_list; -DROP EXTENSION yagp_hooks_collector; +DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql b/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql new file mode 100644 index 00000000000..9abb965db37 --- /dev/null +++ b/gpcontrib/gp_stats_collector/sql/gpsc_utility.sql @@ -0,0 +1,135 @@ +CREATE EXTENSION gp_stats_collector; + +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.enable_utility TO TRUE; +SET gpsc.report_nested_queries TO TRUE; + +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE test_table (a int, b text); +CREATE INDEX test_idx ON test_table(a); +ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; +DROP TABLE test_table; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Partitioning +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE pt_test (a int, b int) +DISTRIBUTED BY (a) +PARTITION BY RANGE (a) +(START (0) END (100) EVERY (50)); +DROP TABLE pt_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Views and Functions +SET gpsc.logging_mode to 'TBL'; + +CREATE VIEW test_view AS SELECT 1 AS a; +CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; +DROP VIEW test_view; +DROP FUNCTION test_func(int); + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Transaction Operations +SET gpsc.logging_mode to 'TBL'; + +BEGIN; +SAVEPOINT sp1; +ROLLBACK TO sp1; +COMMIT; + +BEGIN; +SAVEPOINT sp2; +ABORT; + +BEGIN; +ROLLBACK; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- DML Operations +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE dml_test (a int, b text); +INSERT INTO dml_test VALUES (1, 'test'); +UPDATE dml_test SET b = 'updated' WHERE a = 1; +DELETE FROM dml_test WHERE a = 1; +DROP TABLE dml_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- COPY Operations +SET gpsc.logging_mode to 'TBL'; + +CREATE TABLE copy_test (a int); +COPY (SELECT 1) TO STDOUT; +DROP TABLE copy_test; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- Prepared Statements and error during execute +SET gpsc.logging_mode to 'TBL'; + +PREPARE test_prep(int) AS SELECT $1/0 AS value; +EXECUTE test_prep(0::int); +DEALLOCATE test_prep; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +-- GUC Settings +SET gpsc.logging_mode to 'TBL'; + +SET gpsc.report_nested_queries TO FALSE; +RESET gpsc.report_nested_queries; + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +SELECT gpsc.truncate_log() IS NOT NULL AS t; + +DROP FUNCTION gpsc_status_order(text); +DROP EXTENSION gp_stats_collector; +RESET gpsc.enable; +RESET gpsc.report_nested_queries; +RESET gpsc.enable_utility; +RESET gpsc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp similarity index 79% rename from gpcontrib/yagp_hooks_collector/src/Config.cpp rename to gpcontrib/gp_stats_collector/src/Config.cpp index 62c16e91d1f..e117aa941fd 100644 --- a/gpcontrib/yagp_hooks_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -20,7 +20,7 @@ * Config.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/Config.cpp + * gpcontrib/gp_stats_collector/src/Config.cpp * *------------------------------------------------------------------------- */ @@ -62,63 +62,63 @@ static void assign_ignored_users_hook(const char *, void *) { void Config::init_gucs() { DefineCustomStringVariable( - "yagpcc.uds_path", "Sets filesystem path of the agent socket", 0LL, - &guc_uds_path, "/tmp/yagpcc_agent.sock", PGC_SUSET, + "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, + "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable_analyze", "Collect analyze metrics in yagpcc", 0LL, + "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, &guc_enable_analyze, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.enable_cdbstats", "Collect CDB metrics in yagpcc", 0LL, + "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, &guc_enable_cdbstats, true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( - "yagpcc.report_nested_queries", "Collect stats on nested queries", 0LL, + "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, &guc_report_nested_queries, true, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - DefineCustomStringVariable("yagpcc.ignored_users_list", - "Make yagpcc ignore queries issued by given users", + DefineCustomStringVariable("gpsc.ignored_users_list", + "Make gpsc ignore queries issued by given users", 0LL, &guc_ignored_users, "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, assign_ignored_users_hook, 0LL); DefineCustomIntVariable( - "yagpcc.max_text_size", - "Make yagpcc trim query texts longer than configured size in bytes", NULL, + "gpsc.max_text_size", + "Make gpsc trim query texts longer than configured size in bytes", NULL, &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); DefineCustomIntVariable( - "yagpcc.max_plan_size", - "Make yagpcc trim plan longer than configured size", NULL, + "gpsc.max_plan_size", + "Make gpsc trim plan longer than configured size", NULL, &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); DefineCustomIntVariable( - "yagpcc.min_analyze_time", + "gpsc.min_analyze_time", "Sets the minimum execution time above which plans will be logged.", "Zero prints all plans. -1 turns this feature off.", &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); DefineCustomEnumVariable( - "yagpcc.logging_mode", "Logging mode: UDS or PG Table", NULL, + "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, NULL); DefineCustomBoolVariable( - "yagpcc.enable_utility", "Collect utility statement stats", NULL, + "gpsc.enable_utility", "Collect utility statement stats", NULL, &guc_enable_utility, false, PGC_USERSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } @@ -127,27 +127,27 @@ void Config::update_ignored_users(const char *new_guc_ignored_users) { auto new_ignored_users_set = std::make_unique(); if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { /* Need a modifiable copy of string */ - char *rawstring = ya_gpdb::pstrdup(new_guc_ignored_users); + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); List *elemlist; ListCell *l; /* Parse string into list of identifiers */ - if (!ya_gpdb::split_identifier_string(rawstring, ',', &elemlist)) { + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { /* syntax error in list */ - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); ereport( LOG, (errcode(ERRCODE_SYNTAX_ERROR), errmsg( - "invalid list syntax in parameter yagpcc.ignored_users_list"))); + "invalid list syntax in parameter gpsc.ignored_users_list"))); return; } foreach (l, elemlist) { new_ignored_users_set->insert((char *)lfirst(l)); } - ya_gpdb::pfree(rawstring); - ya_gpdb::list_free(elemlist); + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); } ignored_users_ = std::move(new_ignored_users_set); } diff --git a/gpcontrib/yagp_hooks_collector/src/Config.h b/gpcontrib/gp_stats_collector/src/Config.h similarity index 97% rename from gpcontrib/yagp_hooks_collector/src/Config.h rename to gpcontrib/gp_stats_collector/src/Config.h index 01ae5ea328e..91a1ffe44f2 100644 --- a/gpcontrib/yagp_hooks_collector/src/Config.h +++ b/gpcontrib/gp_stats_collector/src/Config.h @@ -20,7 +20,7 @@ * Config.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/Config.h + * gpcontrib/gp_stats_collector/src/Config.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp similarity index 86% rename from gpcontrib/yagp_hooks_collector/src/EventSender.cpp rename to gpcontrib/gp_stats_collector/src/EventSender.cpp index 6993814ffbf..b28ceba175a 100644 --- a/gpcontrib/yagp_hooks_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -20,7 +20,7 @@ * EventSender.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/EventSender.cpp + * gpcontrib/gp_stats_collector/src/EventSender.cpp * *------------------------------------------------------------------------- */ @@ -106,7 +106,7 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, return true; } -bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, +bool EventSender::log_query_req(const gpsc::SetQueryReq &req, const std::string &event, bool utility) { bool clear_big_fields = false; switch (config.logging_mode()) { @@ -114,7 +114,7 @@ bool EventSender::log_query_req(const yagpcc::SetQueryReq &req, clear_big_fields = UDSConnector::report_query(req, event, config); break; case LOG_MODE_TBL: - ya_gpdb::insert_log(req, utility); + gpdb::insert_log(req, utility); clear_big_fields = false; break; default: @@ -170,7 +170,7 @@ void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { instr_time starttime; INSTR_TIME_SET_CURRENT(starttime); query_desc->showstatctx = - ya_gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); } } } @@ -192,12 +192,12 @@ void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { // context so it will go away at executor_end. if (query_desc->totaltime == NULL) { MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = ya_gpdb::instr_alloc(1, INSTRUMENT_ALL, false); - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); + gpdb::mem_ctx_switch_to(oldcxt); } } - yagpcc::GPMetrics stats; + gpsc::GPMetrics stats; std::swap(stats, *query_msg->mutable_query_metrics()); if (log_query_req(*query_msg, "started", false /* utility */)) { clear_big_fields(query_msg); @@ -233,7 +233,7 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { submit_query(query_desc); auto &query = get_query(query_desc); auto *query_msg = query.message.get(); - *query_msg = create_query_req(yagpcc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); *query_msg->mutable_submit_time() = current_ts(); set_query_info(query_msg); set_qi_nesting_level(query_msg, nesting_level); @@ -256,27 +256,27 @@ void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, QueryMetricsStatus status, bool utility, ErrorData *edata) { - yagpcc::QueryStatus query_status; + gpsc::QueryStatus query_status; std::string msg; switch (status) { case METRICS_QUERY_DONE: case METRICS_INNER_QUERY_DONE: - query_status = yagpcc::QueryStatus::QUERY_STATUS_DONE; + query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; msg = "done"; break; case METRICS_QUERY_ERROR: - query_status = yagpcc::QueryStatus::QUERY_STATUS_ERROR; + query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; msg = "error"; break; case METRICS_QUERY_CANCELING: // at the moment we don't track this event, but I`ll leave this code // here just in case Assert(false); - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELLING; + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; msg = "cancelling"; break; case METRICS_QUERY_CANCELED: - query_status = yagpcc::QueryStatus::QUERY_STATUS_CANCELED; + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; msg = "cancelled"; break; default: @@ -285,15 +285,15 @@ void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, } auto prev_state = query.state; update_query_state(query, QueryState::DONE, utility, - query_status == yagpcc::QueryStatus::QUERY_STATUS_DONE); + query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); auto query_msg = query.message.get(); query_msg->set_query_status(query_status); if (status == METRICS_QUERY_ERROR) { bool error_flushed = elog_message() == NULL; if (error_flushed && (edata == NULL || edata->message == NULL)) { - ereport(WARNING, (errmsg("YAGPCC missing error message"))); + ereport(WARNING, (errmsg("GPSC missing error message"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } else { set_qi_error_message( query_msg, error_flushed ? edata->message : elog_message(), config); @@ -324,18 +324,18 @@ void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, // Skip sending done message if query errored before submit. if (!qdesc_submitted(query_desc)) { if (status != METRICS_QUERY_ERROR) { - ereport(WARNING, (errmsg("YAGPCC trying to process DONE hook for " + ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " "unsubmitted and unerrored query"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } return; } if (queries.empty()) { - ereport(WARNING, (errmsg("YAGPCC cannot find query to process DONE hook"))); + ereport(WARNING, (errmsg("GPSC cannot find query to process DONE hook"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); return; } auto &query = get_query(query_desc); @@ -346,8 +346,8 @@ void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, update_nested_counters(query_desc); queries.erase(QueryKey::from_qdesc(query_desc)); - pfree(query_desc->yagp_query_key); - query_desc->yagp_query_key = NULL; + pfree(query_desc->gpsc_query_key); + query_desc->gpsc_query_key = NULL; } void EventSender::ic_metrics_collect() { @@ -395,7 +395,7 @@ void EventSender::analyze_stats_collect(QueryDesc *query_desc) { } // Make sure stats accumulation is done. // (Note: it's okay if several levels of hook all do this.) - ya_gpdb::instr_end_loop(query_desc->totaltime); + gpdb::instr_end_loop(query_desc->totaltime); double ms = query_desc->totaltime->total * 1000.0; if (ms >= config.min_analyze_time()) { @@ -424,7 +424,7 @@ EventSender::EventSender() { EventSender::~EventSender() { for (const auto &[qkey, _] : queries) { - ereport(LOG, (errmsg("YAGPCC query with missing done event: " + ereport(LOG, (errmsg("GPSC query with missing done event: " "tmid=%d ssid=%d ccnt=%d nlvl=%d", qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); } @@ -440,7 +440,7 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, break; case QueryState::START: if (query.state == QueryState::SUBMIT) { - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_START); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_START); } else { Assert(false); } @@ -449,11 +449,11 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, // Example of below assert triggering: CURSOR closes before ever being // executed Assert(query->state == QueryState::START || // IsAbortInProgress()); - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_END); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_END); break; case QueryState::DONE: Assert(query.state == QueryState::END || !success || utility); - query.message->set_query_status(yagpcc::QueryStatus::QUERY_STATUS_DONE); + query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_DONE); break; default: Assert(false); @@ -464,28 +464,28 @@ void EventSender::update_query_state(QueryItem &query, QueryState new_state, EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { if (!qdesc_submitted(query_desc)) { ereport(WARNING, - (errmsg("YAGPCC attempting to get query that was not submitted"))); + (errmsg("GPSC attempting to get query that was not submitted"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); throw std::runtime_error("Attempting to get query that was not submitted"); } return queries.find(QueryKey::from_qdesc(query_desc))->second; } void EventSender::submit_query(QueryDesc *query_desc) { - if (query_desc->yagp_query_key) { + if (query_desc->gpsc_query_key) { ereport(WARNING, - (errmsg("YAGPCC trying to submit already submitted query"))); + (errmsg("GPSC trying to submit already submitted query"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } QueryKey::register_qkey(query_desc, nesting_level); auto key = QueryKey::from_qdesc(query_desc); auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); if (!inserted) { - ereport(WARNING, (errmsg("YAGPCC duplicate query submit detected"))); + ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); ereport(DEBUG3, - (errmsg("YAGPCC query sourceText: %s", query_desc->sourceText))); + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); } } @@ -498,16 +498,16 @@ void EventSender::update_nested_counters(QueryDesc *query_desc) { if (end_time >= start_time) { nested_timing += end_time - start_time; } else { - ereport(WARNING, (errmsg("YAGPCC query start_time > end_time (%f > %f)", + ereport(WARNING, (errmsg("GPSC query start_time > end_time (%f > %f)", start_time, end_time))); ereport(DEBUG3, - (errmsg("YAGPCC nested query text %s", query_desc->sourceText))); + (errmsg("GPSC nested query text %s", query_desc->sourceText))); } } } bool EventSender::qdesc_submitted(QueryDesc *query_desc) { - if (query_desc->yagp_query_key == NULL) { + if (query_desc->gpsc_query_key == NULL) { return false; } return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); @@ -528,4 +528,4 @@ bool EventSender::filter_query(QueryDesc *query_desc) { } EventSender::QueryItem::QueryItem(QueryState st) - : message(std::make_unique()), state(st) {} + : message(std::make_unique()), state(st) {} diff --git a/gpcontrib/yagp_hooks_collector/src/EventSender.h b/gpcontrib/gp_stats_collector/src/EventSender.h similarity index 84% rename from gpcontrib/yagp_hooks_collector/src/EventSender.h rename to gpcontrib/gp_stats_collector/src/EventSender.h index ef7dcb0bf8c..154c2c0dceb 100644 --- a/gpcontrib/yagp_hooks_collector/src/EventSender.h +++ b/gpcontrib/gp_stats_collector/src/EventSender.h @@ -20,7 +20,7 @@ * EventSender.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/EventSender.h + * gpcontrib/gp_stats_collector/src/EventSender.h * *------------------------------------------------------------------------- */ @@ -45,7 +45,7 @@ extern "C" { class UDSConnector; struct QueryDesc; -namespace yagpcc { +namespace gpsc { class SetQueryReq; } @@ -67,24 +67,24 @@ struct QueryKey { } static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { - query_desc->yagp_query_key = - (YagpQueryKey *)ya_gpdb::palloc0(sizeof(YagpQueryKey)); + query_desc->gpsc_query_key = + (GpscQueryKey *)gpdb::palloc0(sizeof(GpscQueryKey)); int32 tmid; gp_gettmid(&tmid); - query_desc->yagp_query_key->tmid = tmid; - query_desc->yagp_query_key->ssid = gp_session_id; - query_desc->yagp_query_key->ccnt = gp_command_count; - query_desc->yagp_query_key->nesting_level = nesting_level; - query_desc->yagp_query_key->query_desc_addr = (uintptr_t)query_desc; + query_desc->gpsc_query_key->tmid = tmid; + query_desc->gpsc_query_key->ssid = gp_session_id; + query_desc->gpsc_query_key->ccnt = gp_command_count; + query_desc->gpsc_query_key->nesting_level = nesting_level; + query_desc->gpsc_query_key->query_desc_addr = (uintptr_t)query_desc; } static QueryKey from_qdesc(QueryDesc *query_desc) { return { - .tmid = query_desc->yagp_query_key->tmid, - .ssid = query_desc->yagp_query_key->ssid, - .ccnt = query_desc->yagp_query_key->ccnt, - .nesting_level = query_desc->yagp_query_key->nesting_level, - .query_desc_addr = query_desc->yagp_query_key->query_desc_addr, + .tmid = query_desc->gpsc_query_key->tmid, + .ssid = query_desc->gpsc_query_key->ssid, + .ccnt = query_desc->gpsc_query_key->ccnt, + .nesting_level = query_desc->gpsc_query_key->nesting_level, + .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, }; } }; @@ -130,13 +130,13 @@ class EventSender { enum QueryState { SUBMIT, START, END, DONE }; struct QueryItem { - std::unique_ptr message; + std::unique_ptr message; QueryState state; explicit QueryItem(QueryState st); }; - bool log_query_req(const yagpcc::SetQueryReq &req, const std::string &event, + bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, bool utility); bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); void update_query_state(QueryItem &query, QueryState new_state, bool utility, diff --git a/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp b/gpcontrib/gp_stats_collector/src/GpscStat.cpp similarity index 78% rename from gpcontrib/yagp_hooks_collector/src/YagpStat.cpp rename to gpcontrib/gp_stats_collector/src/GpscStat.cpp index 3a760b6ea97..c4029f085cf 100644 --- a/gpcontrib/yagp_hooks_collector/src/YagpStat.cpp +++ b/gpcontrib/gp_stats_collector/src/GpscStat.cpp @@ -17,15 +17,15 @@ * specific language governing permissions and limitations * under the License. * - * YagpStat.cpp + * GpscStat.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/YagpStat.cpp + * gpcontrib/gp_stats_collector/src/GpscStat.cpp * *------------------------------------------------------------------------- */ -#include "YagpStat.h" +#include "GpscStat.h" #include @@ -41,21 +41,21 @@ extern "C" { namespace { struct ProtectedData { slock_t mutex; - YagpStat::Data data; + GpscStat::Data data; }; shmem_startup_hook_type prev_shmem_startup_hook = NULL; ProtectedData *data = nullptr; -void yagp_shmem_startup() { +void gpsc_shmem_startup() { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); bool found; data = reinterpret_cast( - ShmemInitStruct("yagp_stat_messages", sizeof(ProtectedData), &found)); + ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); if (!found) { SpinLockInit(&data->mutex); - data->data = YagpStat::Data(); + data->data = GpscStat::Data(); } LWLockRelease(AddinShmemInitLock); } @@ -70,49 +70,49 @@ class LockGuard { }; } // namespace -void YagpStat::init() { +void GpscStat::init() { if (!process_shared_preload_libraries_in_progress) return; RequestAddinShmemSpace(sizeof(ProtectedData)); prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = yagp_shmem_startup; + shmem_startup_hook = gpsc_shmem_startup; } -void YagpStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } +void GpscStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } -void YagpStat::reset() { +void GpscStat::reset() { LockGuard lg(&data->mutex); - data->data = YagpStat::Data(); + data->data = GpscStat::Data(); } -void YagpStat::report_send(int32_t msg_size) { +void GpscStat::report_send(int32_t msg_size) { LockGuard lg(&data->mutex); data->data.total++; data->data.max_message_size = std::max(msg_size, data->data.max_message_size); } -void YagpStat::report_bad_connection() { +void GpscStat::report_bad_connection() { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_connects++; } -void YagpStat::report_bad_send(int32_t msg_size) { +void GpscStat::report_bad_send(int32_t msg_size) { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_sends++; data->data.max_message_size = std::max(msg_size, data->data.max_message_size); } -void YagpStat::report_error() { +void GpscStat::report_error() { LockGuard lg(&data->mutex); data->data.total++; data->data.failed_other++; } -YagpStat::Data YagpStat::get_stats() { +GpscStat::Data GpscStat::get_stats() { LockGuard lg(&data->mutex); return data->data; } -bool YagpStat::loaded() { return data != nullptr; } +bool GpscStat::loaded() { return data != nullptr; } diff --git a/gpcontrib/yagp_hooks_collector/src/YagpStat.h b/gpcontrib/gp_stats_collector/src/GpscStat.h similarity index 94% rename from gpcontrib/yagp_hooks_collector/src/YagpStat.h rename to gpcontrib/gp_stats_collector/src/GpscStat.h index 57fc90cd4d1..af1a1261776 100644 --- a/gpcontrib/yagp_hooks_collector/src/YagpStat.h +++ b/gpcontrib/gp_stats_collector/src/GpscStat.h @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * YagpStat.h + * GpscStat.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/YagpStat.h + * gpcontrib/gp_stats_collector/src/GpscStat.h * *------------------------------------------------------------------------- */ @@ -29,7 +29,7 @@ #include -class YagpStat { +class GpscStat { public: struct Data { int64_t total, failed_sends, failed_connects, failed_other; diff --git a/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp b/gpcontrib/gp_stats_collector/src/PgUtils.cpp similarity index 83% rename from gpcontrib/yagp_hooks_collector/src/PgUtils.cpp rename to gpcontrib/gp_stats_collector/src/PgUtils.cpp index ed4bf4d7e64..3dbee97061b 100644 --- a/gpcontrib/yagp_hooks_collector/src/PgUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/PgUtils.cpp @@ -20,7 +20,7 @@ * PgUtils.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/PgUtils.cpp + * gpcontrib/gp_stats_collector/src/PgUtils.cpp * *------------------------------------------------------------------------- */ @@ -37,31 +37,31 @@ extern "C" { std::string get_user_name() { // username is allocated on stack, we don't need to pfree it. const char *username = - ya_gpdb::get_config_option("session_authorization", false, false); + gpdb::get_config_option("session_authorization", false, false); return username ? std::string(username) : ""; } std::string get_db_name() { - char *dbname = ya_gpdb::get_database_name(MyDatabaseId); + char *dbname = gpdb::get_database_name(MyDatabaseId); if (dbname) { std::string result(dbname); - ya_gpdb::pfree(dbname); + gpdb::pfree(dbname); return result; } return ""; } std::string get_rg_name() { - auto groupId = ya_gpdb::get_rg_id_by_session_id(MySessionState->sessionId); + auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); if (!OidIsValid(groupId)) return ""; - char *rgname = ya_gpdb::get_rg_name_for_id(groupId); + char *rgname = gpdb::get_rg_name_for_id(groupId); if (rgname == nullptr) return ""; std::string result(rgname); - ya_gpdb::pfree(rgname); + gpdb::pfree(rgname); return result; } @@ -77,7 +77,7 @@ std::string get_rg_name() { * segment. An example would be `select a from tbl where is_good_value(b);`. In * this case master will issue one top-level statement, but segments will change * contexts for UDF execution and execute is_good_value(b) once for each tuple - * as a nested query. Creating massive load on gpcc agent. + * as a nested query. Creating massive load on external agent. * * Hence, here is a decision: * 1) ignore all queries that are nested on segments @@ -87,8 +87,8 @@ std::string get_rg_name() { */ bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - if (query_desc->yagp_query_key == NULL) { + if (query_desc->gpsc_query_key == NULL) { return nesting_level == 0; } - return query_desc->yagp_query_key->nesting_level == 0; + return query_desc->gpsc_query_key->nesting_level == 0; } diff --git a/gpcontrib/yagp_hooks_collector/src/PgUtils.h b/gpcontrib/gp_stats_collector/src/PgUtils.h similarity index 96% rename from gpcontrib/yagp_hooks_collector/src/PgUtils.h rename to gpcontrib/gp_stats_collector/src/PgUtils.h index 5113fadbff2..d9f673e7cbc 100644 --- a/gpcontrib/yagp_hooks_collector/src/PgUtils.h +++ b/gpcontrib/gp_stats_collector/src/PgUtils.h @@ -20,7 +20,7 @@ * PgUtils.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/PgUtils.h + * gpcontrib/gp_stats_collector/src/PgUtils.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp b/gpcontrib/gp_stats_collector/src/ProcStats.cpp similarity index 92% rename from gpcontrib/yagp_hooks_collector/src/ProcStats.cpp rename to gpcontrib/gp_stats_collector/src/ProcStats.cpp index 72a12e8ca00..9c557879fc6 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProcStats.cpp +++ b/gpcontrib/gp_stats_collector/src/ProcStats.cpp @@ -20,13 +20,13 @@ * ProcStats.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProcStats.cpp + * gpcontrib/gp_stats_collector/src/ProcStats.cpp * *------------------------------------------------------------------------- */ #include "ProcStats.h" -#include "yagpcc_metrics.pb.h" +#include "gpsc_metrics.pb.h" #include #include #include @@ -42,7 +42,7 @@ namespace { proc_stat >> tmp >> stat_name; \ stats->set_##stat_name(stat_name - stats->stat_name()); -void fill_io_stats(yagpcc::SystemStat *stats) { +void fill_io_stats(gpsc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/io"); std::string tmp; FILL_IO_STAT(rchar); @@ -54,7 +54,7 @@ void fill_io_stats(yagpcc::SystemStat *stats) { FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(yagpcc::SystemStat *stats) { +void fill_cpu_stats(gpsc::SystemStat *stats) { static const int UTIME_ID = 13; static const int STIME_ID = 14; static const int VSIZE_ID = 22; @@ -92,7 +92,7 @@ void fill_cpu_stats(yagpcc::SystemStat *stats) { } } -void fill_status_stats(yagpcc::SystemStat *stats) { +void fill_status_stats(gpsc::SystemStat *stats) { std::ifstream proc_stat("/proc/self/status"); std::string key, measure; while (proc_stat >> key) { @@ -118,7 +118,7 @@ void fill_status_stats(yagpcc::SystemStat *stats) { } } // namespace -void fill_self_stats(yagpcc::SystemStat *stats) { +void fill_self_stats(gpsc::SystemStat *stats) { fill_io_stats(stats); fill_cpu_stats(stats); fill_status_stats(stats); diff --git a/gpcontrib/yagp_hooks_collector/src/ProcStats.h b/gpcontrib/gp_stats_collector/src/ProcStats.h similarity index 89% rename from gpcontrib/yagp_hooks_collector/src/ProcStats.h rename to gpcontrib/gp_stats_collector/src/ProcStats.h index 7629edd0aea..4473125f875 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProcStats.h +++ b/gpcontrib/gp_stats_collector/src/ProcStats.h @@ -20,15 +20,15 @@ * ProcStats.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProcStats.h + * gpcontrib/gp_stats_collector/src/ProcStats.h * *------------------------------------------------------------------------- */ #pragma once -namespace yagpcc { +namespace gpsc { class SystemStat; } -void fill_self_stats(yagpcc::SystemStat *stats); \ No newline at end of file +void fill_self_stats(gpsc::SystemStat *stats); \ No newline at end of file diff --git a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp similarity index 85% rename from gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp rename to gpcontrib/gp_stats_collector/src/ProtoUtils.cpp index b449ae20900..c9ceff4739b 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp @@ -20,7 +20,7 @@ * ProtoUtils.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProtoUtils.cpp + * gpcontrib/gp_stats_collector/src/ProtoUtils.cpp * *------------------------------------------------------------------------- */ @@ -74,7 +74,7 @@ google::protobuf::Timestamp current_ts() { return current_ts; } -void set_query_key(yagpcc::QueryKey *key) { +void set_query_key(gpsc::QueryKey *key) { key->set_ccnt(gp_command_count); key->set_ssid(gp_session_id); int32 tmid = 0; @@ -82,7 +82,7 @@ void set_query_key(yagpcc::QueryKey *key) { key->set_tmid(tmid); } -void set_segment_key(yagpcc::SegmentKey *key) { +void set_segment_key(gpsc::SegmentKey *key) { key->set_dbid(GpIdentity.dbid); key->set_segindex(GpIdentity.segindex); } @@ -109,51 +109,51 @@ std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { return std::string(str, cut_pos); } -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { auto qi = req->mutable_query_info(); qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? yagpcc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : yagpcc::PlanGenerator::PLAN_GENERATOR_PLANNER); + ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_explain_state(query_desc, true); + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); if (es.str) { *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); - StringInfo norm_plan = ya_gpdb::gen_normplan(es.str->data); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); if (norm_plan) { *qi->mutable_template_plan_text() = trim_str_shrink_utf8( norm_plan->data, norm_plan->len, config.max_plan_size()); qi->set_plan_id( hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - ya_gpdb::pfree(norm_plan->data); + gpdb::pfree(norm_plan->data); } qi->set_query_id(query_desc->plannedstmt->queryId); - ya_gpdb::pfree(es.str->data); + gpdb::pfree(es.str->data); } - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(oldcxt); } } -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config) { if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { auto qi = req->mutable_query_info(); *qi->mutable_query_text() = trim_str_shrink_utf8( query_desc->sourceText, strlen(query_desc->sourceText), config.max_text_size()); - char *norm_query = ya_gpdb::gen_normquery(query_desc->sourceText); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); if (norm_query) { *qi->mutable_template_query_text() = trim_str_shrink_utf8( norm_query, strlen(norm_query), config.max_text_size()); - ya_gpdb::pfree(norm_query); + gpdb::pfree(norm_query); } } } -void clear_big_fields(yagpcc::SetQueryReq *req) { +void clear_big_fields(gpsc::SetQueryReq *req) { if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->clear_plan_text(); @@ -164,7 +164,7 @@ void clear_big_fields(yagpcc::SetQueryReq *req) { } } -void set_query_info(yagpcc::SetQueryReq *req) { +void set_query_info(gpsc::SetQueryReq *req) { if (Gp_role == GP_ROLE_DISPATCH) { auto qi = req->mutable_query_info(); qi->set_username(get_user_name()); @@ -174,24 +174,24 @@ void set_query_info(yagpcc::SetQueryReq *req) { } } -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level) { +void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) { auto aqi = req->mutable_add_info(); aqi->set_nested_level(nesting_level); } -void set_qi_slice_id(yagpcc::SetQueryReq *req) { +void set_qi_slice_id(gpsc::SetQueryReq *req) { auto aqi = req->mutable_add_info(); aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, +void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, const Config &config) { auto aqi = req->mutable_add_info(); *aqi->mutable_error_message() = trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } -void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, +void set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, QueryDesc *query_desc, int nested_calls, double nested_time) { auto instrument = query_desc->planstate->instrument; @@ -233,7 +233,7 @@ void set_metric_instrumentation(yagpcc::MetricInstrumentation *metrics, metrics->set_inherited_time(nested_time); } -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, +void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time) { if (query_desc->planstate && query_desc->planstate->instrument) { set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, @@ -256,7 +256,7 @@ void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, metrics->mutable_interconnect()->proto_name() <= \ ic_statistics->stat_name) -void set_ic_stats(yagpcc::MetricInstrumentation *metrics, +void set_ic_stats(gpsc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics) { #ifdef IC_TEARDOWN_HOOK UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); @@ -279,8 +279,8 @@ void set_ic_stats(yagpcc::MetricInstrumentation *metrics, #endif } -yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status) { - yagpcc::SetQueryReq req; +gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status) { + gpsc::SetQueryReq req; req.set_query_status(status); *req.mutable_datetime() = current_ts(); set_query_key(req.mutable_query_key()); @@ -292,7 +292,7 @@ double protots_to_double(const google::protobuf::Timestamp &ts) { return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, +void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, const Config &config) { // Make sure it is a valid txn and it is not an utility // statement for ExplainPrintPlan() later. @@ -300,10 +300,10 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, return; } MemoryContext oldcxt = - ya_gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = ya_gpdb::get_analyze_state( + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state( query_desc, query_desc->instrument_options && config.enable_analyze()); - ya_gpdb::mem_ctx_switch_to(oldcxt); + gpdb::mem_ctx_switch_to(oldcxt); if (es.str) { // Remove last line break. if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { @@ -312,6 +312,6 @@ void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *req, auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); req->mutable_query_info()->set_analyze_text(trimmed_analyze); - ya_gpdb::pfree(es.str->data); + gpdb::pfree(es.str->data); } } diff --git a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h b/gpcontrib/gp_stats_collector/src/ProtoUtils.h similarity index 65% rename from gpcontrib/yagp_hooks_collector/src/ProtoUtils.h rename to gpcontrib/gp_stats_collector/src/ProtoUtils.h index c954545494f..5ddcd42d308 100644 --- a/gpcontrib/yagp_hooks_collector/src/ProtoUtils.h +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.h @@ -20,35 +20,35 @@ * ProtoUtils.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/ProtoUtils.h + * gpcontrib/gp_stats_collector/src/ProtoUtils.h * *------------------------------------------------------------------------- */ #pragma once -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" struct QueryDesc; struct ICStatistics; class Config; google::protobuf::Timestamp current_ts(); -void set_query_plan(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config); -void set_query_text(yagpcc::SetQueryReq *req, QueryDesc *query_desc, +void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, const Config &config); -void clear_big_fields(yagpcc::SetQueryReq *req); -void set_query_info(yagpcc::SetQueryReq *req); -void set_qi_nesting_level(yagpcc::SetQueryReq *req, int nesting_level); -void set_qi_slice_id(yagpcc::SetQueryReq *req); -void set_qi_error_message(yagpcc::SetQueryReq *req, const char *err_msg, +void clear_big_fields(gpsc::SetQueryReq *req); +void set_query_info(gpsc::SetQueryReq *req); +void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level); +void set_qi_slice_id(gpsc::SetQueryReq *req); +void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, const Config &config); -void set_gp_metrics(yagpcc::GPMetrics *metrics, QueryDesc *query_desc, +void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, int nested_calls, double nested_time); -void set_ic_stats(yagpcc::MetricInstrumentation *metrics, +void set_ic_stats(gpsc::MetricInstrumentation *metrics, const ICStatistics *ic_statistics); -yagpcc::SetQueryReq create_query_req(yagpcc::QueryStatus status); +gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); -void set_analyze_plan_text(QueryDesc *query_desc, yagpcc::SetQueryReq *message, +void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *message, const Config &config); diff --git a/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp similarity index 87% rename from gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp rename to gpcontrib/gp_stats_collector/src/UDSConnector.cpp index d13a82a5ca9..9a01d4033d0 100644 --- a/gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -20,14 +20,14 @@ * UDSConnector.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/UDSConnector.cpp + * gpcontrib/gp_stats_collector/src/UDSConnector.cpp * *------------------------------------------------------------------------- */ #include "UDSConnector.h" #include "Config.h" -#include "YagpStat.h" +#include "GpscStat.h" #include "memory/gpdbwrappers.h" #include "log/LogOps.h" @@ -44,14 +44,14 @@ extern "C" { #include "postgres.h" } -static void inline log_tracing_failure(const yagpcc::SetQueryReq &req, +static void inline log_tracing_failure(const gpsc::SetQueryReq &req, const std::string &event) { ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", req.query_key().tmid(), req.query_key().ssid(), req.query_key().ccnt(), event.c_str()))); } -bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, +bool UDSConnector::report_query(const gpsc::SetQueryReq &req, const std::string &event, const Config &config) { sockaddr_un address{}; @@ -60,7 +60,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (uds_path.size() >= sizeof(address.sun_path)) { ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); - YagpStat::report_error(); + GpscStat::report_error(); return false; } strcpy(address.sun_path, uds_path.c_str()); @@ -68,7 +68,7 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); if (sockfd == -1) { log_tracing_failure(req, event); - YagpStat::report_error(); + GpscStat::report_error(); return false; } @@ -83,24 +83,24 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, // visible to an end-user and admins. ereport(WARNING, (errmsg("Unable to create non-blocking socket connection %m"))); - YagpStat::report_error(); + GpscStat::report_error(); return false; } if (connect(sockfd, reinterpret_cast(&address), sizeof(address)) == -1) { log_tracing_failure(req, event); - YagpStat::report_bad_connection(); + GpscStat::report_bad_connection(); return false; } const auto data_size = req.ByteSizeLong(); const auto total_size = data_size + sizeof(uint32_t); - auto *buf = static_cast(ya_gpdb::palloc(total_size)); + auto *buf = static_cast(gpdb::palloc(total_size)); // Free buf automatically on error path. struct BufGuard { void *p; - ~BufGuard() { ya_gpdb::pfree(p); } + ~BufGuard() { gpdb::pfree(p); } } buf_guard{buf}; *reinterpret_cast(buf) = data_size; @@ -121,10 +121,10 @@ bool UDSConnector::report_query(const yagpcc::SetQueryReq &req, if (sent < 0) { log_tracing_failure(req, event); - YagpStat::report_bad_send(total_size); + GpscStat::report_bad_send(total_size); return false; } - YagpStat::report_send(total_size); + GpscStat::report_send(total_size); return true; } diff --git a/gpcontrib/yagp_hooks_collector/src/UDSConnector.h b/gpcontrib/gp_stats_collector/src/UDSConnector.h similarity index 88% rename from gpcontrib/yagp_hooks_collector/src/UDSConnector.h rename to gpcontrib/gp_stats_collector/src/UDSConnector.h index be5ab1ef413..a91d22f9df1 100644 --- a/gpcontrib/yagp_hooks_collector/src/UDSConnector.h +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.h @@ -20,19 +20,19 @@ * UDSConnector.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/UDSConnector.h + * gpcontrib/gp_stats_collector/src/UDSConnector.h * *------------------------------------------------------------------------- */ #pragma once -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" class Config; class UDSConnector { public: - bool static report_query(const yagpcc::SetQueryReq &req, + bool static report_query(const gpsc::SetQueryReq &req, const std::string &event, const Config &config); }; diff --git a/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c similarity index 79% rename from gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c rename to gpcontrib/gp_stats_collector/src/gp_stats_collector.c index 271bceee178..d930f72246d 100644 --- a/gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * yagp_hooks_collector.c + * gp_stats_collector.c * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/yagp_hooks_collector.c + * gpcontrib/gp_stats_collector/src/gp_stats_collector.c * *------------------------------------------------------------------------- */ @@ -36,14 +36,14 @@ PG_MODULE_MAGIC; void _PG_init(void); void _PG_fini(void); -PG_FUNCTION_INFO_V1(yagp_stat_messages_reset); -PG_FUNCTION_INFO_V1(yagp_stat_messages); -PG_FUNCTION_INFO_V1(yagp_init_log); -PG_FUNCTION_INFO_V1(yagp_truncate_log); +PG_FUNCTION_INFO_V1(gpsc_stat_messages_reset); +PG_FUNCTION_INFO_V1(gpsc_stat_messages); +PG_FUNCTION_INFO_V1(gpsc_init_log); +PG_FUNCTION_INFO_V1(gpsc_truncate_log); -PG_FUNCTION_INFO_V1(yagp_test_uds_start_server); -PG_FUNCTION_INFO_V1(yagp_test_uds_receive); -PG_FUNCTION_INFO_V1(yagp_test_uds_stop_server); +PG_FUNCTION_INFO_V1(gpsc_test_uds_start_server); +PG_FUNCTION_INFO_V1(gpsc_test_uds_receive); +PG_FUNCTION_INFO_V1(gpsc_test_uds_stop_server); void _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) @@ -55,23 +55,23 @@ void _PG_fini(void) { hooks_deinit(); } -Datum yagp_stat_messages_reset(PG_FUNCTION_ARGS) { +Datum gpsc_stat_messages_reset(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { funcctx = SRF_FIRSTCALL_INIT(); - yagp_functions_reset(); + gpsc_functions_reset(); } funcctx = SRF_PERCALL_SETUP(); SRF_RETURN_DONE(funcctx); } -Datum yagp_stat_messages(PG_FUNCTION_ARGS) { - return yagp_functions_get(fcinfo); +Datum gpsc_stat_messages(PG_FUNCTION_ARGS) { + return gpsc_functions_get(fcinfo); } -Datum yagp_init_log(PG_FUNCTION_ARGS) { +Datum gpsc_init_log(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -83,7 +83,7 @@ Datum yagp_init_log(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_truncate_log(PG_FUNCTION_ARGS) { +Datum gpsc_truncate_log(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -95,7 +95,7 @@ Datum yagp_truncate_log(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_start_server(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { @@ -109,7 +109,7 @@ Datum yagp_test_uds_start_server(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_receive(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; int64 *result; @@ -137,7 +137,7 @@ Datum yagp_test_uds_receive(PG_FUNCTION_ARGS) { SRF_RETURN_DONE(funcctx); } -Datum yagp_test_uds_stop_server(PG_FUNCTION_ARGS) { +Datum gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; if (SRF_IS_FIRSTCALL()) { diff --git a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp similarity index 84% rename from gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp rename to gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index cb4970d60d9..0a40b4cb359 100644 --- a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -20,7 +20,7 @@ * hook_wrappers.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/hook_wrappers.cpp + * gpcontrib/gp_stats_collector/src/hook_wrappers.cpp * *------------------------------------------------------------------------- */ @@ -48,7 +48,7 @@ extern "C" { #undef typeid #include "Config.h" -#include "YagpStat.h" +#include "GpscStat.h" #include "EventSender.h" #include "hook_wrappers.h" #include "memory/gpdbwrappers.h" @@ -67,20 +67,20 @@ static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; #endif static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; -static void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, +static void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags); +static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, uint64 count, bool execute_once); -static void ya_ExecutorFinish_hook(QueryDesc *query_desc); -static void ya_ExecutorEnd_hook(QueryDesc *query_desc); -static void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg); +static void gpsc_ExecutorFinish_hook(QueryDesc *query_desc); +static void gpsc_ExecutorEnd_hook(QueryDesc *query_desc); +static void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg); #ifdef IC_TEARDOWN_HOOK -static void ya_ic_teardown_hook(ChunkTransportState *transportStates, +static void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors); #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -static void ya_analyze_stats_collect_hook(QueryDesc *query_desc); +static void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, +static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, @@ -108,34 +108,34 @@ R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { try { return (obj->*func)(args...); } catch (const std::exception &e) { - ereport(FATAL, (errmsg("Unexpected exception in yagpcc %s", e.what()))); + ereport(FATAL, (errmsg("Unexpected exception in gpsc %s", e.what()))); } } void hooks_init() { Config::init_gucs(); - YagpStat::init(); + GpscStat::init(); previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = ya_ExecutorStart_hook; + ExecutorStart_hook = gpsc_ExecutorStart_hook; previous_ExecutorRun_hook = ExecutorRun_hook; - ExecutorRun_hook = ya_ExecutorRun_hook; + ExecutorRun_hook = gpsc_ExecutorRun_hook; previous_ExecutorFinish_hook = ExecutorFinish_hook; - ExecutorFinish_hook = ya_ExecutorFinish_hook; + ExecutorFinish_hook = gpsc_ExecutorFinish_hook; previous_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = ya_ExecutorEnd_hook; + ExecutorEnd_hook = gpsc_ExecutorEnd_hook; previous_query_info_collect_hook = query_info_collect_hook; - query_info_collect_hook = ya_query_info_collect_hook; + query_info_collect_hook = gpsc_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK previous_ic_teardown_hook = ic_teardown_hook; - ic_teardown_hook = ya_ic_teardown_hook; + ic_teardown_hook = gpsc_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK previous_analyze_stats_collect_hook = analyze_stats_collect_hook; - analyze_stats_collect_hook = ya_analyze_stats_collect_hook; + analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; #endif stat_statements_parser_init(); previous_ProcessUtility_hook = ProcessUtility_hook; - ProcessUtility_hook = ya_process_utility_hook; + ProcessUtility_hook = gpsc_process_utility_hook; } void hooks_deinit() { @@ -154,11 +154,11 @@ void hooks_deinit() { if (sender) { delete sender; } - YagpStat::deinit(); + GpscStat::deinit(); ProcessUtility_hook = previous_ProcessUtility_hook; } -void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { +void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, eflags); if (previous_ExecutorStart_hook) { @@ -170,7 +170,7 @@ void ya_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { eflags); } -void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, +void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, uint64 count, bool execute_once) { get_sender()->incr_depth(); PG_TRY(); @@ -189,7 +189,7 @@ void ya_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, PG_END_TRY(); } -void ya_ExecutorFinish_hook(QueryDesc *query_desc) { +void gpsc_ExecutorFinish_hook(QueryDesc *query_desc) { get_sender()->incr_depth(); PG_TRY(); { @@ -207,7 +207,7 @@ void ya_ExecutorFinish_hook(QueryDesc *query_desc) { PG_END_TRY(); } -void ya_ExecutorEnd_hook(QueryDesc *query_desc) { +void gpsc_ExecutorEnd_hook(QueryDesc *query_desc) { cpp_call(get_sender(), &EventSender::executor_end, query_desc); if (previous_ExecutorEnd_hook) { (*previous_ExecutorEnd_hook)(query_desc); @@ -216,7 +216,7 @@ void ya_ExecutorEnd_hook(QueryDesc *query_desc) { } } -void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { +void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) { cpp_call(get_sender(), &EventSender::query_metrics_collect, status, arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); if (previous_query_info_collect_hook) { @@ -225,7 +225,7 @@ void ya_query_info_collect_hook(QueryMetricsStatus status, void *arg) { } #ifdef IC_TEARDOWN_HOOK -void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { +void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { cpp_call(get_sender(), &EventSender::ic_metrics_collect); if (previous_ic_teardown_hook) { (*previous_ic_teardown_hook)(transportStates, hasErrors); @@ -234,7 +234,7 @@ void ya_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { +void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) { cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); if (previous_analyze_stats_collect_hook) { (*previous_analyze_stats_collect_hook)(query_desc); @@ -242,7 +242,7 @@ void ya_analyze_stats_collect_hook(QueryDesc *query_desc) { } #endif -static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, +static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, @@ -296,22 +296,22 @@ static void ya_process_utility_hook(PlannedStmt *pstmt, const char *queryString, } static void check_stats_loaded() { - if (!YagpStat::loaded()) { + if (!GpscStat::loaded()) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("yagp_hooks_collector must be loaded via " + errmsg("gp_stats_collector must be loaded via " "shared_preload_libraries"))); } } -void yagp_functions_reset() { +void gpsc_functions_reset() { check_stats_loaded(); - YagpStat::reset(); + GpscStat::reset(); } -Datum yagp_functions_get(FunctionCallInfo fcinfo) { +Datum gpsc_functions_get(FunctionCallInfo fcinfo) { const int ATTNUM = 6; check_stats_loaded(); - auto stats = YagpStat::get_stats(); + auto stats = GpscStat::get_stats(); TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, 0 /* attdim */); @@ -335,7 +335,7 @@ Datum yagp_functions_get(FunctionCallInfo fcinfo) { values[3] = Int64GetDatum(stats.failed_connects); values[4] = Int64GetDatum(stats.failed_other); values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = ya_gpdb::heap_form_tuple(tupdesc, values, nulls); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); Datum result = HeapTupleGetDatum(tuple); PG_RETURN_DATUM(result); } diff --git a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h b/gpcontrib/gp_stats_collector/src/hook_wrappers.h similarity index 89% rename from gpcontrib/yagp_hooks_collector/src/hook_wrappers.h rename to gpcontrib/gp_stats_collector/src/hook_wrappers.h index 443406a5259..06c8d064404 100644 --- a/gpcontrib/yagp_hooks_collector/src/hook_wrappers.h +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.h @@ -20,7 +20,7 @@ * hook_wrappers.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/hook_wrappers.h + * gpcontrib/gp_stats_collector/src/hook_wrappers.h * *------------------------------------------------------------------------- */ @@ -33,8 +33,8 @@ extern "C" { extern void hooks_init(); extern void hooks_deinit(); -extern void yagp_functions_reset(); -extern Datum yagp_functions_get(FunctionCallInfo fcinfo); +extern void gpsc_functions_reset(); +extern Datum gpsc_functions_get(FunctionCallInfo fcinfo); extern void init_log(); extern void truncate_log(); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp similarity index 91% rename from gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp rename to gpcontrib/gp_stats_collector/src/log/LogOps.cpp index e8c927ece84..ef4f39c0749 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp @@ -20,12 +20,12 @@ * LogOps.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogOps.cpp + * gpcontrib/gp_stats_collector/src/log/LogOps.cpp * *------------------------------------------------------------------------- */ -#include "protos/yagpcc_set_service.pb.h" +#include "protos/gpsc_set_service.pb.h" #include "LogOps.h" #include "LogSchema.h" @@ -82,14 +82,14 @@ void init_log() { /* Table can be dropped only via DROP EXTENSION */ recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); } else { - ereport(NOTICE, (errmsg("YAGPCC failed to create log table or schema"))); + ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); } /* Make changes visible */ CommandCounterIncrement(); } -void insert_log(const yagpcc::SetQueryReq &req, bool utility) { +void insert_log(const gpsc::SetQueryReq &req, bool utility) { Oid namespaceId; Oid relationId; Relation rel; @@ -112,15 +112,15 @@ void insert_log(const yagpcc::SetQueryReq &req, bool utility) { return; } - bool nulls[natts_yagp_log]; - Datum values[natts_yagp_log]; + bool nulls[natts_gpsc_log]; + Datum values[natts_gpsc_log]; memset(nulls, true, sizeof(nulls)); memset(values, 0, sizeof(values)); extract_query_req(req, "", values, nulls); - nulls[attnum_yagp_log_utility] = false; - values[attnum_yagp_log_utility] = BoolGetDatum(utility); + nulls[attnum_gpsc_log_utility] = false; + values[attnum_gpsc_log_utility] = BoolGetDatum(utility); rel = heap_open(relationId, RowExclusiveLock); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogOps.h b/gpcontrib/gp_stats_collector/src/log/LogOps.h similarity index 83% rename from gpcontrib/yagp_hooks_collector/src/log/LogOps.h rename to gpcontrib/gp_stats_collector/src/log/LogOps.h index 1fc30c21030..f784270bb8f 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogOps.h +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.h @@ -20,7 +20,7 @@ * LogOps.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogOps.h + * gpcontrib/gp_stats_collector/src/log/LogOps.h * *------------------------------------------------------------------------- */ @@ -35,12 +35,12 @@ extern "C" { } extern "C" { -/* CREATE TABLE yagpcc.__log (...); */ +/* CREATE TABLE gpsc.__log (...); */ void init_log(); -/* TRUNCATE yagpcc.__log */ +/* TRUNCATE gpsc.__log */ void truncate_log(); } -/* INSERT INTO yagpcc.__log VALUES (...) */ -void insert_log(const yagpcc::SetQueryReq &req, bool utility); +/* INSERT INTO gpsc.__log VALUES (...) */ +void insert_log(const gpsc::SetQueryReq &req, bool utility); diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp similarity index 94% rename from gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp rename to gpcontrib/gp_stats_collector/src/log/LogSchema.cpp index a391b1a2209..f9f43fac2fd 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp @@ -20,7 +20,7 @@ * LogSchema.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogSchema.cpp + * gpcontrib/gp_stats_collector/src/log/LogSchema.cpp * *------------------------------------------------------------------------- */ @@ -36,7 +36,7 @@ const std::unordered_map &proto_name_to_col_idx() { std::unordered_map map; map.reserve(log_tbl_desc.size()); - for (size_t idx = 0; idx < natts_yagp_log; ++idx) { + for (size_t idx = 0; idx < natts_gpsc_log; ++idx) { map.emplace(log_tbl_desc[idx].proto_field_name, idx); } @@ -46,9 +46,9 @@ const std::unordered_map &proto_name_to_col_idx() { } TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_yagp_log); + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); - for (size_t anum = 1; anum <= natts_yagp_log; ++anum) { + for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) { TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, 0 /* attdim */); @@ -104,7 +104,7 @@ void process_field(const google::protobuf::FieldDescriptor *field, if (it == proto_idx_map.end()) { ereport(NOTICE, - (errmsg("YAGPCC protobuf field %s is not registered in log table", + (errmsg("GPSC protobuf field %s is not registered in log table", field_name.c_str()))); return; } diff --git a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h b/gpcontrib/gp_stats_collector/src/log/LogSchema.h similarity index 98% rename from gpcontrib/yagp_hooks_collector/src/log/LogSchema.h rename to gpcontrib/gp_stats_collector/src/log/LogSchema.h index f78acec7ce9..8754741823a 100644 --- a/gpcontrib/yagp_hooks_collector/src/log/LogSchema.h +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.h @@ -20,7 +20,7 @@ * LogSchema.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/log/LogSchema.h + * gpcontrib/gp_stats_collector/src/log/LogSchema.h * *------------------------------------------------------------------------- */ @@ -50,7 +50,7 @@ class Timestamp; } // namespace protobuf } // namespace google -inline constexpr std::string_view schema_name = "yagpcc"; +inline constexpr std::string_view schema_name = "gpsc"; inline constexpr std::string_view log_relname = "__log"; struct LogDesc { @@ -165,8 +165,8 @@ inline constexpr std::array log_tbl_desc = { }; /* clang-format on */ -inline constexpr size_t natts_yagp_log = log_tbl_desc.size(); -inline constexpr size_t attnum_yagp_log_utility = natts_yagp_log - 1; +inline constexpr size_t natts_gpsc_log = log_tbl_desc.size(); +inline constexpr size_t attnum_gpsc_log_utility = natts_gpsc_log - 1; const std::unordered_map &proto_name_to_col_idx(); diff --git a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp similarity index 81% rename from gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp rename to gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp index 22083e8bdaf..4e3f6dae99f 100644 --- a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp @@ -20,7 +20,7 @@ * gpdbwrappers.cpp * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.cpp + * gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp * *------------------------------------------------------------------------- */ @@ -125,22 +125,22 @@ auto wrap_noexcept(Func &&func, Args &&...args) noexcept } } // namespace -void *ya_gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } +void *gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } -void *ya_gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } +void *gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } -char *ya_gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } +char *gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } -char *ya_gpdb::get_database_name(Oid dbid) noexcept { +char *gpdb::get_database_name(Oid dbid) noexcept { return wrap_noexcept(::get_database_name, dbid); } -bool ya_gpdb::split_identifier_string(char *rawstring, char separator, +bool gpdb::split_identifier_string(char *rawstring, char separator, List **namelist) noexcept { return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, +ExplainState gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept { return wrap_noexcept([&]() { ExplainState *es = NewExplainState(); @@ -154,7 +154,7 @@ ExplainState ya_gpdb::get_explain_state(QueryDesc *query_desc, }); } -ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, +ExplainState gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept { return wrap_noexcept([&]() { ExplainState *es = NewExplainState(); @@ -174,12 +174,12 @@ ExplainState ya_gpdb::get_analyze_state(QueryDesc *query_desc, }); } -Instrumentation *ya_gpdb::instr_alloc(size_t n, int instrument_options, +Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options, bool async_mode) { return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } -HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, +HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull) { if (!tupleDescriptor || !values || !isnull) throw std::runtime_error( @@ -188,7 +188,7 @@ HeapTuple ya_gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void ya_gpdb::pfree(void *pointer) noexcept { +void gpdb::pfree(void *pointer) noexcept { // Note that ::pfree asserts that pointer != NULL. if (!pointer) return; @@ -196,11 +196,11 @@ void ya_gpdb::pfree(void *pointer) noexcept { wrap_noexcept(::pfree, pointer); } -MemoryContext ya_gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { +MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { return MemoryContextSwitchTo(context); } -const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, +const char *gpdb::get_config_option(const char *name, bool missing_ok, bool restrict_superuser) noexcept { if (!name) return nullptr; @@ -208,7 +208,7 @@ const char *ya_gpdb::get_config_option(const char *name, bool missing_ok, return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void ya_gpdb::list_free(List *list) noexcept { +void gpdb::list_free(List *list) noexcept { if (!list) return; @@ -216,7 +216,7 @@ void ya_gpdb::list_free(List *list) noexcept { } CdbExplain_ShowStatCtx * -ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime) { if (!query_desc) throw std::runtime_error("Invalid query descriptor"); @@ -224,29 +224,29 @@ ya_gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void ya_gpdb::instr_end_loop(Instrumentation *instr) { +void gpdb::instr_end_loop(Instrumentation *instr) { if (!instr) throw std::runtime_error("Invalid instrumentation pointer"); wrap_throw(::InstrEndLoop, instr); } -char *ya_gpdb::gen_normquery(const char *query) noexcept { +char *gpdb::gen_normquery(const char *query) noexcept { return wrap_noexcept(::gen_normquery, query); } -StringInfo ya_gpdb::gen_normplan(const char *exec_plan) noexcept { +StringInfo gpdb::gen_normplan(const char *exec_plan) noexcept { return wrap_noexcept(::gen_normplan, exec_plan); } -char *ya_gpdb::get_rg_name_for_id(Oid group_id) { +char *gpdb::get_rg_name_for_id(Oid group_id) { return wrap_throw(GetResGroupNameForId, group_id); } -Oid ya_gpdb::get_rg_id_by_session_id(int session_id) { +Oid gpdb::get_rg_id_by_session_id(int session_id) { return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } -void ya_gpdb::insert_log(const yagpcc::SetQueryReq &req, bool utility) { +void gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) { return wrap_throw(::insert_log, req, utility); } diff --git a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h similarity index 92% rename from gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h rename to gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h index fe9b3ba0487..576007f6c7c 100644 --- a/gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h @@ -20,7 +20,7 @@ * gpdbwrappers.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/memory/gpdbwrappers.h + * gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h * *------------------------------------------------------------------------- */ @@ -43,11 +43,11 @@ extern "C" { #include #include -namespace yagpcc { +namespace gpsc { class SetQueryReq; -} // namespace yagpcc +} // namespace gpsc -namespace ya_gpdb { +namespace gpdb { // Functions that call palloc(). // Make sure correct memory context is set. @@ -68,7 +68,7 @@ void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query) noexcept; StringInfo gen_normplan(const char *executionPlan) noexcept; char *get_rg_name_for_id(Oid group_id); -void insert_log(const yagpcc::SetQueryReq &req, bool utility); +void insert_log(const gpsc::SetQueryReq &req, bool utility); // Palloc-free functions. void pfree(void *pointer) noexcept; @@ -78,4 +78,4 @@ const char *get_config_option(const char *name, bool missing_ok, void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace ya_gpdb +} // namespace gpdb diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md b/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md new file mode 100644 index 00000000000..927189474fe --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/README.md @@ -0,0 +1,20 @@ + + +This directory contains a slightly modified subset of pg_stat_statements for PG v9.4 to be used in query and plan ID generation. diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c similarity index 99% rename from gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c index 7404208055f..e24f53536a4 100644 --- a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c @@ -20,7 +20,7 @@ * pg_stat_statements_ya_parser.c * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h similarity index 93% rename from gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h index 96c6a776dba..a613ba04259 100644 --- a/gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h @@ -20,7 +20,7 @@ * pg_stat_statements_ya_parser.h * * IDENTIFICATION - * gpcontrib/yagp_hooks_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h * *------------------------------------------------------------------------- */ diff --git a/gpcontrib/yagp_hooks_collector/README.md b/gpcontrib/yagp_hooks_collector/README.md deleted file mode 100644 index 9f465a190cb..00000000000 --- a/gpcontrib/yagp_hooks_collector/README.md +++ /dev/null @@ -1,28 +0,0 @@ -## YAGP Hooks Collector - -An extension for collecting greenplum query execution metrics and reporting them to an external agent. - -### Collected Statistics - -#### 1. Query Lifecycle -- **What:** Captures query text, normalized query text, timestamps (submit, start, end, done), and user/database info. -- **GUC:** `yagpcc.enable`. - -#### 2. `EXPLAIN` data -- **What:** Triggers generation of the `EXPLAIN (TEXT, COSTS, VERBOSE)` and captures it. -- **GUC:** `yagpcc.enable`. - -#### 3. `EXPLAIN ANALYZE` data -- **What:** Triggers generation of the `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)` and captures it. -- **GUCs:** `yagpcc.enable`, `yagpcc.min_analyze_time`, `yagpcc.enable_cdbstats`(ANALYZE), `yagpcc.enable_analyze`(BUFFERS, TIMING, VERBOSE). - -#### 4. Other Metrics -- **What:** Captures Instrument, Greenplum, System, Network, Interconnect, Spill metrics. -- **GUC:** `yagpcc.enable`. - -### General Configuration -- **Nested Queries:** When `yagpcc.report_nested_queries` is `false`, only top-level queries are reported from the coordinator and segments, when `true`, both top-level and nested queries are reported from the coordinator, from segments collected as aggregates. -- **Data Destination:** All collected data is sent to a Unix Domain Socket. Configure the path with `yagpcc.uds_path`. -- **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `yagpcc.ignored_users_list`. -- **Trimming plans:** Query texts and execution plans are trimmed based on `yagpcc.max_text_size` and `yagpcc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. -- **Analyze collection:** Analyze is sent if execution time exceeds `yagpcc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `yagpcc.enable_analyze` is true. diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out b/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out deleted file mode 100644 index 6689b6a4ed3..00000000000 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_locale.out +++ /dev/null @@ -1,23 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. --- start_ignore -DROP DATABASE IF EXISTS yagp_test_locale; --- end_ignore -CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c yagp_test_locale -CREATE EXTENSION yagp_hooks_collector; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.enable TO TRUE; -CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -DROP TABLE yagp_hi_안녕세계; -DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out b/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out deleted file mode 100644 index d04929ffb4a..00000000000 --- a/gpcontrib/yagp_hooks_collector/expected/yagp_uds.out +++ /dev/null @@ -1,42 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; --- end_ignore -\set UDS_PATH '/tmp/yagpcc_test.sock' --- Configure extension to send via UDS -SET yagpcc.uds_path TO :'UDS_PATH'; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.logging_mode TO 'UDS'; --- Start receiver -SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); - __test_uds_start_server -------------------------- -(0 rows) - --- Send -SELECT 1; - ?column? ----------- - 1 -(1 row) - --- Receive -SELECT yagpcc.__test_uds_receive() > 0 as received; - received ----------- - t -(1 row) - --- Stop receiver -SELECT yagpcc.__test_uds_stop_server(); - __test_uds_stop_server ------------------------- -(0 rows) - --- Cleanup -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.uds_path; -RESET yagpcc.ignored_users_list; -RESET yagpcc.enable; -RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql deleted file mode 100644 index f56351e0d43..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_cursors.sql +++ /dev/null @@ -1,85 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; - --- DECLARE -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_0 CURSOR FOR SELECT 0; -CLOSE cursor_stats_0; -COMMIT; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- DECLARE WITH HOLD -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; -CLOSE cursor_stats_1; -DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; -CLOSE cursor_stats_2; -COMMIT; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- ROLLBACK -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_3 CURSOR FOR SELECT 1; -CLOSE cursor_stats_3; -DECLARE cursor_stats_4 CURSOR FOR SELECT 1; -ROLLBACK; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- FETCH -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; -DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; -FETCH 1 IN cursor_stats_5; -FETCH 1 IN cursor_stats_6; -CLOSE cursor_stats_5; -CLOSE cursor_stats_6; -COMMIT; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql deleted file mode 100644 index 65d867d1680..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_locale.sql +++ /dev/null @@ -1,29 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. - --- start_ignore -DROP DATABASE IF EXISTS yagp_test_locale; --- end_ignore - -CREATE DATABASE yagp_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c yagp_test_locale - -CREATE EXTENSION yagp_hooks_collector; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.enable TO TRUE; - -CREATE TABLE yagp_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO yagp_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE yagp_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; - -RESET yagpcc.enable; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; -DROP TABLE yagp_hi_안녕세계; -DROP EXTENSION yagp_hooks_collector; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql deleted file mode 100644 index 90e972ae4c1..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_select.sql +++ /dev/null @@ -1,69 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE OR REPLACE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; -SET yagpcc.enable_utility TO FALSE; - --- Basic SELECT tests -SET yagpcc.logging_mode to 'TBL'; - -SELECT 1; -SELECT COUNT(*) FROM generate_series(1,10); - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Transaction test -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -SELECT 1; -COMMIT; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- CTE test -SET yagpcc.logging_mode to 'TBL'; - -WITH t AS (VALUES (1), (2)) -SELECT * FROM t; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Prepared statement test -SET yagpcc.logging_mode to 'TBL'; - -PREPARE test_stmt AS SELECT 1; -EXECUTE test_stmt; -DEALLOCATE test_stmt; - -RESET yagpcc.logging_mode; -SELECT segid, query_text, query_status FROM yagpcc.log ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql deleted file mode 100644 index 3eef697a4e7..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_uds.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS yagp_hooks_collector; --- end_ignore - -\set UDS_PATH '/tmp/yagpcc_test.sock' - --- Configure extension to send via UDS -SET yagpcc.uds_path TO :'UDS_PATH'; -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.logging_mode TO 'UDS'; - --- Start receiver -SELECT yagpcc.__test_uds_start_server(:'UDS_PATH'); - --- Send -SELECT 1; - --- Receive -SELECT yagpcc.__test_uds_receive() > 0 as received; - --- Stop receiver -SELECT yagpcc.__test_uds_stop_server(); - --- Cleanup -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.uds_path; -RESET yagpcc.ignored_users_list; -RESET yagpcc.enable; -RESET yagpcc.logging_mode; diff --git a/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql b/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql deleted file mode 100644 index cf9c1d253d0..00000000000 --- a/gpcontrib/yagp_hooks_collector/sql/yagp_utility.sql +++ /dev/null @@ -1,135 +0,0 @@ -CREATE EXTENSION yagp_hooks_collector; - -CREATE OR REPLACE FUNCTION yagp_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; - -SET yagpcc.ignored_users_list TO ''; -SET yagpcc.enable TO TRUE; -SET yagpcc.enable_utility TO TRUE; -SET yagpcc.report_nested_queries TO TRUE; - -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE test_table (a int, b text); -CREATE INDEX test_idx ON test_table(a); -ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; -DROP TABLE test_table; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Partitioning -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE pt_test (a int, b int) -DISTRIBUTED BY (a) -PARTITION BY RANGE (a) -(START (0) END (100) EVERY (50)); -DROP TABLE pt_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Views and Functions -SET yagpcc.logging_mode to 'TBL'; - -CREATE VIEW test_view AS SELECT 1 AS a; -CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; -DROP VIEW test_view; -DROP FUNCTION test_func(int); - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Transaction Operations -SET yagpcc.logging_mode to 'TBL'; - -BEGIN; -SAVEPOINT sp1; -ROLLBACK TO sp1; -COMMIT; - -BEGIN; -SAVEPOINT sp2; -ABORT; - -BEGIN; -ROLLBACK; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- DML Operations -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE dml_test (a int, b text); -INSERT INTO dml_test VALUES (1, 'test'); -UPDATE dml_test SET b = 'updated' WHERE a = 1; -DELETE FROM dml_test WHERE a = 1; -DROP TABLE dml_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- COPY Operations -SET yagpcc.logging_mode to 'TBL'; - -CREATE TABLE copy_test (a int); -COPY (SELECT 1) TO STDOUT; -DROP TABLE copy_test; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- Prepared Statements and error during execute -SET yagpcc.logging_mode to 'TBL'; - -PREPARE test_prep(int) AS SELECT $1/0 AS value; -EXECUTE test_prep(0::int); -DEALLOCATE test_prep; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - --- GUC Settings -SET yagpcc.logging_mode to 'TBL'; - -SET yagpcc.report_nested_queries TO FALSE; -RESET yagpcc.report_nested_queries; - -RESET yagpcc.logging_mode; - -SELECT segid, query_text, query_status FROM yagpcc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, yagp_status_order(query_status) ASC; -SELECT yagpcc.truncate_log() IS NOT NULL AS t; - -DROP FUNCTION yagp_status_order(text); -DROP EXTENSION yagp_hooks_collector; -RESET yagpcc.enable; -RESET yagpcc.report_nested_queries; -RESET yagpcc.enable_utility; -RESET yagpcc.ignored_users_list; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql deleted file mode 100644 index 8684ca73915..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0--1.1.sql +++ /dev/null @@ -1,113 +0,0 @@ -/* yagp_hooks_collector--1.0--1.1.sql */ - --- complain if script is sourced in psql, rather than via ALTER EXTENSION -\echo Use "ALTER EXTENSION yagp_hooks_collector UPDATE TO '1.1'" to load this file. \quit - -CREATE SCHEMA yagpcc; - --- Unlink existing objects from extension. -ALTER EXTENSION yagp_hooks_collector DROP VIEW yagp_stat_messages; -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION yagp_stat_messages_reset(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_segments(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_f_on_master(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); -ALTER EXTENSION yagp_hooks_collector DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); - --- Now drop the objects. -DROP VIEW yagp_stat_messages; -DROP FUNCTION yagp_stat_messages_reset(); -DROP FUNCTION __yagp_stat_messages_f_on_segments(); -DROP FUNCTION __yagp_stat_messages_f_on_master(); -DROP FUNCTION __yagp_stat_messages_reset_f_on_segments(); -DROP FUNCTION __yagp_stat_messages_reset_f_on_master(); - --- Recreate functions and view in new schema. -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT yagpcc.__stat_messages_reset_f_on_master(); - SELECT yagpcc.__stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagpcc.stat_messages AS - SELECT C.* - FROM yagpcc.__stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM yagpcc.__stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; - --- Create new objects. -CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - --- Creates log table inside yagpcc schema. -SELECT yagpcc.__init_log_on_master(); -SELECT yagpcc.__init_log_on_segments(); - -CREATE VIEW yagpcc.log AS - SELECT * FROM yagpcc.__log -- master - UNION ALL - SELECT * FROM gp_dist_random('yagpcc.__log') -- segments - ORDER BY tmid, ssid, ccnt; - -CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.truncate_log() -RETURNS SETOF void AS $$ -BEGIN - PERFORM yagpcc.__truncate_log_on_master(); - PERFORM yagpcc.__truncate_log_on_segments(); -END; -$$ LANGUAGE plpgsql VOLATILE; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql deleted file mode 100644 index 270cab92382..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.0.sql +++ /dev/null @@ -1,55 +0,0 @@ -/* yagp_hooks_collector--1.0.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit - -CREATE FUNCTION __yagp_stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagp_stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT __yagp_stat_messages_reset_f_on_master(); - SELECT __yagp_stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION __yagp_stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagp_stat_messages AS - SELECT C.* - FROM __yagp_stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM __yagp_stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql deleted file mode 100644 index 83bfb553638..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector--1.1.sql +++ /dev/null @@ -1,110 +0,0 @@ -/* yagp_hooks_collector--1.1.sql */ - --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION yagp_hooks_collector" to load this file. \quit - -CREATE SCHEMA yagpcc; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_reset_f_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_stat_messages_reset' -LANGUAGE C EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.stat_messages_reset() -RETURNS SETOF void -AS -$$ - SELECT yagpcc.__stat_messages_reset_f_on_master(); - SELECT yagpcc.__stat_messages_reset_f_on_segments(); -$$ -LANGUAGE SQL EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_master() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__stat_messages_f_on_segments() -RETURNS SETOF record -AS 'MODULE_PATHNAME', 'yagp_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE VIEW yagpcc.stat_messages AS - SELECT C.* - FROM yagpcc.__stat_messages_f_on_master() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) - UNION ALL - SELECT C.* - FROM yagpcc.__stat_messages_f_on_segments() as C ( - segid int, - total_messages bigint, - send_failures bigint, - connection_failures bigint, - other_errors bigint, - max_message_size int - ) -ORDER BY segid; - -CREATE FUNCTION yagpcc.__init_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__init_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - --- Creates log table inside yagpcc schema. -SELECT yagpcc.__init_log_on_master(); -SELECT yagpcc.__init_log_on_segments(); - -CREATE VIEW yagpcc.log AS - SELECT * FROM yagpcc.__log -- master - UNION ALL - SELECT * FROM gp_dist_random('yagpcc.__log') -- segments -ORDER BY tmid, ssid, ccnt; - -CREATE FUNCTION yagpcc.__truncate_log_on_master() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__truncate_log_on_segments() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; - -CREATE FUNCTION yagpcc.truncate_log() -RETURNS SETOF void AS $$ -BEGIN - PERFORM yagpcc.__truncate_log_on_master(); - PERFORM yagpcc.__truncate_log_on_segments(); -END; -$$ LANGUAGE plpgsql VOLATILE; - -CREATE FUNCTION yagpcc.__test_uds_start_server(path text) -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_test_uds_start_server' -LANGUAGE C STRICT EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__test_uds_receive(timeout_ms int DEFAULT 2000) -RETURNS SETOF bigint -AS 'MODULE_PATHNAME', 'yagp_test_uds_receive' -LANGUAGE C STRICT EXECUTE ON MASTER; - -CREATE FUNCTION yagpcc.__test_uds_stop_server() -RETURNS SETOF void -AS 'MODULE_PATHNAME', 'yagp_test_uds_stop_server' -LANGUAGE C EXECUTE ON MASTER; diff --git a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control b/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control deleted file mode 100644 index cb5906a1302..00000000000 --- a/gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control +++ /dev/null @@ -1,5 +0,0 @@ -# yagp_hooks_collector extension -comment = 'Intercept query and plan execution hooks and report them to Yandex GPCC agents' -default_version = '1.1' -module_pathname = '$libdir/yagp_hooks_collector' -superuser = true diff --git a/pom.xml b/pom.xml index 45e62756b11..dbc67b99a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -154,12 +154,6 @@ code or new licensing patterns. gpcontrib/gp_exttable_fdw/gp_exttable_fdw.control gpcontrib/diskquota/** - gpcontrib/yagp_hooks_collector/yagp_hooks_collector.control - gpcontrib/yagp_hooks_collector/protos/yagpcc_set_service.proto - gpcontrib/yagp_hooks_collector/protos/yagpcc_plan.proto - gpcontrib/yagp_hooks_collector/protos/yagpcc_metrics.proto - gpcontrib/yagp_hooks_collector/.clang-format - gpcontrib/yagp_hooks_collector/Makefile getversion .git-blame-ignore-revs @@ -1277,6 +1271,16 @@ code or new licensing patterns. src/include/task/task_states.h src/include/task/job_metadata.h + + gpcontrib/gp_stats_collector/gp_stats_collector.control + gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto + gpcontrib/gp_stats_collector/protos/gpsc_plan.proto + gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto + gpcontrib/gp_stats_collector/.clang-format + gpcontrib/gp_stats_collector/Makefile + diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 553830e8599..0ea5874e884 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -376,7 +376,7 @@ PortalCleanup(Portal portal) CurrentResourceOwner = saveResourceOwner; } else { /* GPDB hook for collecting query info */ - if (queryDesc->yagp_query_key && query_info_collect_hook) + if (queryDesc->gpsc_query_key && query_info_collect_hook) (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); } } diff --git a/src/backend/tcop/pquery.c b/src/backend/tcop/pquery.c index 7c1dbc480bc..e5512bb8271 100644 --- a/src/backend/tcop/pquery.c +++ b/src/backend/tcop/pquery.c @@ -127,8 +127,8 @@ CreateQueryDesc(PlannedStmt *plannedstmt, if (Gp_role != GP_ROLE_EXECUTE) increment_command_count(); - /* null this field until set by YAGP Hooks collector */ - qd->yagp_query_key = NULL; + /* null this field until set by GP Stats Collector */ + qd->gpsc_query_key = NULL; return qd; } diff --git a/src/include/executor/execdesc.h b/src/include/executor/execdesc.h index e469945a4c5..d50d3e48f6b 100644 --- a/src/include/executor/execdesc.h +++ b/src/include/executor/execdesc.h @@ -22,14 +22,14 @@ struct CdbExplain_ShowStatCtx; /* private, in "cdb/cdbexplain.c" */ -typedef struct YagpQueryKey +typedef struct GpscQueryKey { int tmid; /* transaction time */ int ssid; /* session id */ int ccnt; /* command count */ int nesting_level; uintptr_t query_desc_addr; -} YagpQueryKey; +} GpscQueryKey; /* * SerializedParams is used to serialize external query parameters @@ -339,8 +339,8 @@ typedef struct QueryDesc /* This is always set NULL by the core system, but plugins can change it */ struct Instrumentation *totaltime; /* total time spent in ExecutorRun */ - /* YAGP Hooks collector */ - YagpQueryKey *yagp_query_key; + /* GP Stats Collector */ + GpscQueryKey *gpsc_query_key; } QueryDesc; /* in pquery.c */ From 6c94eae707c9d4985864e43da7fcd1b8993cd173 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Thu, 26 Mar 2026 11:21:35 +0300 Subject: [PATCH 102/167] [gp_stats_collector] Simplify Makefile and add -Wno-unused-but-set-variable --- gpcontrib/gp_stats_collector/Makefile | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/gpcontrib/gp_stats_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile index c8f7b3c30fe..43255ca1955 100644 --- a/gpcontrib/gp_stats_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -10,13 +10,8 @@ C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/*/*.cpp)) OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) -override CXXFLAGS = -Werror -fPIC -g3 -Wall -Wpointer-arith -Wendif-labels \ - -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv \ - -Wno-unused-but-set-variable -Wno-address -Wno-format-truncation \ - -Wno-stringop-truncation -g -ggdb -std=c++17 -Iinclude -Isrc/protos -Isrc -DGPBUILD - -PG_CXXFLAGS += -Isrc -Iinclude -SHLIB_LINK += -lprotobuf -lpthread -lstdc++ +PG_CXXFLAGS += -Werror -Wall -Wno-unused-but-set-variable -std=c++17 -Isrc/protos -Isrc -Iinclude -DGPBUILD +SHLIB_LINK += -lprotobuf -lstdc++ EXTRA_CLEAN = src/protos ifdef USE_PGXS @@ -30,10 +25,11 @@ include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -src/protos/%.pb.cpp src/protos/%.pb.h: protos/%.proto +src/protos/.done: $(wildcard protos/*.proto) @mkdir -p src/protos protoc -I /usr/include -I /usr/local/include -I . --cpp_out=src $^ - mv src/protos/$*.pb.cc src/protos/$*.pb.cpp + for f in src/protos/*.pb.cc; do mv "$$f" "$${f%.cc}.cpp"; done + touch $@ -$(CPP_OBJS): src/protos/gpsc_metrics.pb.h src/protos/gpsc_plan.pb.h src/protos/gpsc_set_service.pb.h -src/protos/gpsc_set_service.pb.o: src/protos/gpsc_metrics.pb.h +src/protos/%.pb.cpp src/protos/%.pb.h: src/protos/.done ; +$(CPP_OBJS): src/protos/.done From bea3819787fc76d9762971db2a3536489c4a8f35 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 08:34:53 +0300 Subject: [PATCH 103/167] [gp_stats_collector] Build by default with extension disabled via GUCs Enable building gp_stats_collector by default in configure. Add missing check in verify_query() to ensure the extension does not execute main code while disabled. Always verify protobuf version once the shared library is preloaded. --- .github/workflows/build-cloudberry-rocky8.yml | 3 +-- .github/workflows/build-cloudberry.yml | 3 +-- .github/workflows/build-deb-cloudberry.yml | 3 +-- .../cloudberry/scripts/configure-cloudberry.sh | 1 + gpcontrib/gp_stats_collector/src/Config.cpp | 6 +++--- gpcontrib/gp_stats_collector/src/EventSender.cpp | 16 +++++++++------- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index f009539d37d..433363d6bc6 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -545,11 +545,10 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 94b38529d21..4cff875836f 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -541,11 +541,10 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 592ef2eaf69..5d458c46e13 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -453,14 +453,13 @@ jobs: shell: bash env: SRC_DIR: ${{ github.workspace }} - CONFIGURE_EXTRA_OPTS: --with-gp-stats-collector run: | set -eo pipefail export BUILD_DESTINATION=${SRC_DIR}/debian/build chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} CONFIGURE_EXTRA_OPTS=${{ env.CONFIGURE_EXTRA_OPTS }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then echo "::error::Configure script failed" exit 1 fi diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index d30a0b794f0..a9086a434fb 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -163,6 +163,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ + --with-gp-stats-collector \ --with-gssapi \ --with-ldap \ --with-libxml \ diff --git a/gpcontrib/gp_stats_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp index e117aa941fd..2f40b30e922 100644 --- a/gpcontrib/gp_stats_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -40,7 +40,7 @@ extern "C" { static char *guc_uds_path = nullptr; static bool guc_enable_analyze = true; static bool guc_enable_cdbstats = true; -static bool guc_enable_collector = true; +static bool guc_enable_collector = false; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; static int guc_max_text_size = 1 << 20; // in bytes (1MB) @@ -68,7 +68,7 @@ void Config::init_gucs() { DefineCustomBoolVariable( "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, - true, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + false, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); DefineCustomBoolVariable( "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, @@ -88,7 +88,7 @@ void Config::init_gucs() { DefineCustomStringVariable("gpsc.ignored_users_list", "Make gpsc ignore queries issued by given users", 0LL, &guc_ignored_users, - "gpadmin,repl,gpperfmon,monitor", PGC_SUSET, + "", PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, assign_ignored_users_hook, 0LL); diff --git a/gpcontrib/gp_stats_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp index b28ceba175a..c0faaf0ad0e 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -68,6 +68,10 @@ bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, // not executed yet, causing DONE to be skipped/added. config.sync(); + if (!config.enable_collector()) { + return false; + } + if (utility && !config.enable_utility()) { return false; } @@ -409,13 +413,11 @@ EventSender::EventSender() { // Perform initial sync to get default GUC values config.sync(); - if (config.enable_collector()) { - try { - GOOGLE_PROTOBUF_VERIFY_VERSION; - proto_verified = true; - } catch (const std::exception &e) { - ereport(INFO, (errmsg("Unable to start query tracing %s", e.what()))); - } + try { + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; + } catch (const std::exception &e) { + ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", e.what()))); } #ifdef IC_TEARDOWN_HOOK memset(&ic_statistics, 0, sizeof(ICStatistics)); From 0cc2cd1add60e7ceb40f245ca813721a5643f997 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 08:47:13 +0300 Subject: [PATCH 104/167] [gp_stats_collector] Code quality cleanup Delete stale .gitignore. Add Apache headers to .proto files. Change #pragma once to #ifndef guards. Remove test result files from tree. Change ereport(FATAL) to ereport(ERROR). Remove internal naming suffixes. Apply clang-format from gporca. --- .gitignore | 2 +- gpcontrib/gp_stats_collector/.clang-format | 180 +++- gpcontrib/gp_stats_collector/.gitignore | 5 - .../protos/gpsc_metrics.proto | 18 + .../gp_stats_collector/protos/gpsc_plan.proto | 18 + .../protos/gpsc_set_service.proto | 18 + .../results/gpsc_cursors.out | 163 --- .../gp_stats_collector/results/gpsc_dist.out | 175 ---- .../results/gpsc_guc_cache.out | 61 -- .../results/gpsc_locale.out | 23 - .../results/gpsc_select.out | 136 --- .../gp_stats_collector/results/gpsc_uds.out | 42 - .../results/gpsc_utf8_trim.out | 68 -- .../results/gpsc_utility.out | 248 ----- gpcontrib/gp_stats_collector/src/Config.cpp | 248 ++--- gpcontrib/gp_stats_collector/src/Config.h | 98 +- .../gp_stats_collector/src/EventSender.cpp | 976 ++++++++++-------- .../gp_stats_collector/src/EventSender.h | 243 +++-- gpcontrib/gp_stats_collector/src/GpscStat.cpp | 142 ++- gpcontrib/gp_stats_collector/src/GpscStat.h | 36 +- gpcontrib/gp_stats_collector/src/PgUtils.cpp | 68 +- .../gp_stats_collector/src/ProcStats.cpp | 179 ++-- gpcontrib/gp_stats_collector/src/ProcStats.h | 9 +- .../gp_stats_collector/src/ProtoUtils.cpp | 492 +++++---- gpcontrib/gp_stats_collector/src/ProtoUtils.h | 17 +- .../gp_stats_collector/src/UDSConnector.cpp | 166 +-- .../gp_stats_collector/src/UDSConnector.h | 12 +- .../src/gp_stats_collector.c | 169 +-- .../gp_stats_collector/src/hook_wrappers.cpp | 627 ++++++----- .../gp_stats_collector/src/hook_wrappers.h | 6 +- .../gp_stats_collector/src/log/LogOps.cpp | 199 ++-- gpcontrib/gp_stats_collector/src/log/LogOps.h | 5 +- .../gp_stats_collector/src/log/LogSchema.cpp | 261 ++--- .../gp_stats_collector/src/log/LogSchema.h | 38 +- .../src/memory/gpdbwrappers.cpp | 386 ++++--- .../src/memory/gpdbwrappers.h | 33 +- ...a_parser.c => pg_stat_statements_parser.c} | 94 +- ...a_parser.h => pg_stat_statements_parser.h} | 12 +- pom.xml | 3 - 39 files changed, 2774 insertions(+), 2902 deletions(-) delete mode 100644 gpcontrib/gp_stats_collector/.gitignore delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_cursors.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_dist.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_locale.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_select.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_uds.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out delete mode 100644 gpcontrib/gp_stats_collector/results/gpsc_utility.out rename gpcontrib/gp_stats_collector/src/stat_statements_parser/{pg_stat_statements_ya_parser.c => pg_stat_statements_parser.c} (82%) rename gpcontrib/gp_stats_collector/src/stat_statements_parser/{pg_stat_statements_ya_parser.h => pg_stat_statements_parser.h} (87%) diff --git a/.gitignore b/.gitignore index 7f5110d5c8e..5c21989c4ab 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,4 @@ lib*.pc /compile_commands.json /tmp_install/ /.cache/ -/install/ +/install/ \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/.clang-format b/gpcontrib/gp_stats_collector/.clang-format index 99130575c9a..eb90ff33671 100644 --- a/gpcontrib/gp_stats_collector/.clang-format +++ b/gpcontrib/gp_stats_collector/.clang-format @@ -1,2 +1,178 @@ -BasedOnStyle: LLVM -SortIncludes: false +--- +Language: Cpp +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false +AlignConsecutiveAssignments: false +AlignConsecutiveBitFields: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: None +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: All +AlwaysBreakAfterReturnType: AllDefinitions +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: true + AfterClass: true + AfterControlStatement: Always + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterObjCDeclaration: true + AfterStruct: true + AfterUnion: true + AfterExternBlock: false + BeforeCatch: true + BeforeElse: true + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Custom +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: false +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^<.*' + Priority: 1 + - Regex: '"protos/.*\.pb\.h"' + Priority: 2 + - Regex: '"postgres\.h"' + Priority: 3 + - Regex: '.*' + Priority: 4 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IncludeIsMainSourceRegex: '' +IndentCaseLabels: true +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 3 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Right +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + - ParseTestProto + - ParsePartialTestProto + CanonicalDelimiter: '' + BasedOnStyle: google +ReflowComments: false +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: true +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseCRLF: false +UseTab: Always +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE +... + + diff --git a/gpcontrib/gp_stats_collector/.gitignore b/gpcontrib/gp_stats_collector/.gitignore deleted file mode 100644 index e8dfe855dad..00000000000 --- a/gpcontrib/gp_stats_collector/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.o -*.so -src/protos/ -.vscode -compile_commands.json diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index a9e26471839..7853dc58db7 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; package gpsc; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto index 5a7269edd20..c1632478464 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_plan.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; package gpsc; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto index 4cd795424ab..bcf09074ed7 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + syntax = "proto3"; import "google/protobuf/timestamp.proto"; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out b/gpcontrib/gp_stats_collector/results/gpsc_cursors.out deleted file mode 100644 index 282d9ac49e1..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_cursors.out +++ /dev/null @@ -1,163 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.report_nested_queries TO TRUE; --- DECLARE -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_0 CURSOR FOR SELECT 0; -CLOSE cursor_stats_0; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_0 CURSOR FOR SELECT 0; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_0; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_0; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- DECLARE WITH HOLD -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; -CLOSE cursor_stats_1; -DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; -CLOSE cursor_stats_2; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT 1; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_1; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_1; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_2; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_2; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(14 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- ROLLBACK -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_3 CURSOR FOR SELECT 1; -CLOSE cursor_stats_3; -DECLARE cursor_stats_4 CURSOR FOR SELECT 1; -ROLLBACK; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_3 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_3; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_3; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_4 CURSOR FOR SELECT 1; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(12 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- FETCH -SET gpsc.logging_mode to 'TBL'; -BEGIN; -DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; -DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; -FETCH 1 IN cursor_stats_5; - ?column? ----------- - 2 -(1 row) - -FETCH 1 IN cursor_stats_6; - ?column? ----------- - 3 -(1 row) - -CLOSE cursor_stats_5; -CLOSE cursor_stats_6; -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_5 CURSOR WITH HOLD FOR SELECT 2; | QUERY_STATUS_DONE - -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_SUBMIT - -1 | DECLARE cursor_stats_6 CURSOR WITH HOLD FOR SELECT 3; | QUERY_STATUS_DONE - -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_SUBMIT - -1 | FETCH 1 IN cursor_stats_5; | QUERY_STATUS_DONE - -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_SUBMIT - -1 | FETCH 1 IN cursor_stats_6; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_5; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_5; | QUERY_STATUS_DONE - -1 | CLOSE cursor_stats_6; | QUERY_STATUS_SUBMIT - -1 | CLOSE cursor_stats_6; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(18 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_dist.out b/gpcontrib/gp_stats_collector/results/gpsc_dist.out deleted file mode 100644 index 92e8678767b..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_dist.out +++ /dev/null @@ -1,175 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.enable_utility TO FALSE; --- Hash distributed table -CREATE TABLE test_hash_dist (id int) DISTRIBUTED BY (id); -INSERT INTO test_hash_dist SELECT 1; -SET gpsc.logging_mode to 'TBL'; -SET optimizer_enable_direct_dispatch TO TRUE; --- Direct dispatch is used here, only one segment is scanned. -select * from test_hash_dist where id = 1; - id ----- - 1 -(1 row) - -RESET optimizer_enable_direct_dispatch; -RESET gpsc.logging_mode; --- Should see 8 rows. -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------------------+--------------------- - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_SUBMIT - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_START - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_END - -1 | select * from test_hash_dist where id = 1; | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -SET gpsc.logging_mode to 'TBL'; --- Scan all segments. -select * from test_hash_dist; - id ----- - 1 -(1 row) - -DROP TABLE test_hash_dist; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------+--------------------- - -1 | select * from test_hash_dist; | QUERY_STATUS_SUBMIT - -1 | select * from test_hash_dist; | QUERY_STATUS_START - -1 | select * from test_hash_dist; | QUERY_STATUS_END - -1 | select * from test_hash_dist; | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE - 2 | | QUERY_STATUS_SUBMIT - 2 | | QUERY_STATUS_START - 2 | | QUERY_STATUS_END - 2 | | QUERY_STATUS_DONE - | | QUERY_STATUS_SUBMIT - | | QUERY_STATUS_START - | | QUERY_STATUS_END - | | QUERY_STATUS_DONE -(16 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Replicated table -CREATE FUNCTION force_segments() RETURNS SETOF text AS $$ -BEGIN - RETURN NEXT 'seg'; -END; -$$ LANGUAGE plpgsql VOLATILE EXECUTE ON ALL SEGMENTS; -CREATE TABLE test_replicated (id int) DISTRIBUTED REPLICATED; -INSERT INTO test_replicated SELECT 1; -SET gpsc.logging_mode to 'TBL'; -SELECT COUNT(*) FROM test_replicated, force_segments(); - count -------- - 3 -(1 row) - -DROP TABLE test_replicated; -DROP FUNCTION force_segments(); -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------------------+--------------------- - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_SUBMIT - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_START - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_END - -1 | SELECT COUNT(*) FROM test_replicated, force_segments(); | QUERY_STATUS_DONE - 1 | | QUERY_STATUS_SUBMIT - 1 | | QUERY_STATUS_START - 1 | | QUERY_STATUS_END - 1 | | QUERY_STATUS_DONE - 2 | | QUERY_STATUS_SUBMIT - 2 | | QUERY_STATUS_START - 2 | | QUERY_STATUS_END - 2 | | QUERY_STATUS_DONE - | | QUERY_STATUS_SUBMIT - | | QUERY_STATUS_START - | | QUERY_STATUS_END - | | QUERY_STATUS_DONE -(16 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Partially distributed table (2 numsegments) -SET allow_system_table_mods = ON; -CREATE TABLE test_partial_dist (id int, data text) DISTRIBUTED BY (id); -UPDATE gp_distribution_policy SET numsegments = 2 WHERE localoid = 'test_partial_dist'::regclass; -INSERT INTO test_partial_dist SELECT * FROM generate_series(1, 100); -SET gpsc.logging_mode to 'TBL'; -SELECT COUNT(*) FROM test_partial_dist; - count -------- - 100 -(1 row) - -RESET gpsc.logging_mode; -DROP TABLE test_partial_dist; -RESET allow_system_table_mods; --- Should see 12 rows. -SELECT query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - query_text | query_status ------------------------------------------+--------------------- - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_SUBMIT - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_START - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_END - SELECT COUNT(*) FROM test_partial_dist; | QUERY_STATUS_DONE - | QUERY_STATUS_SUBMIT - | QUERY_STATUS_START - | QUERY_STATUS_END - | QUERY_STATUS_DONE - | QUERY_STATUS_SUBMIT - | QUERY_STATUS_START - | QUERY_STATUS_END - | QUERY_STATUS_DONE -(12 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out b/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out deleted file mode 100644 index 19c4774575d..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_guc_cache.out +++ /dev/null @@ -1,61 +0,0 @@ --- --- Test GUC caching for query lifecycle consistency. --- --- The extension logs SUBMIT and DONE events for each query. --- GUC values that control logging (enable_utility, ignored_users_list, ...) --- must be cached at SUBMIT time to ensure DONE uses the same filtering --- criteria. Otherwise, a SET command that modifies these GUCs would --- have its DONE event rejected, creating orphaned SUBMIT entries. --- This is due to query being actually executed between SUBMIT and DONE. --- start_ignore -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; -SELECT gpsc.truncate_log(); - truncate_log --------------- -(0 rows) - --- end_ignore -CREATE OR REPLACE FUNCTION print_last_query(query text) -RETURNS TABLE(query_status text) AS $$ - SELECT query_status - FROM gpsc.log - WHERE segid = -1 AND query_text = query - ORDER BY ccnt DESC -$$ LANGUAGE sql; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.logging_mode TO 'TBL'; --- SET below disables utility logging and DONE must still be logged. -SET gpsc.enable_utility TO FALSE; -SELECT * FROM print_last_query('SET gpsc.enable_utility TO FALSE;'); - query_status ---------------------- - QUERY_STATUS_SUBMIT - QUERY_STATUS_DONE -(2 rows) - --- SELECT below adds current user to ignore list and DONE must still be logged. --- start_ignore -SELECT set_config('gpsc.ignored_users_list', current_user, false); - set_config ------------- - gpadmin -(1 row) - --- end_ignore -SELECT * FROM print_last_query('SELECT set_config(''gpsc.ignored_users_list'', current_user, false);'); - query_status ---------------------- - QUERY_STATUS_SUBMIT - QUERY_STATUS_START - QUERY_STATUS_END - QUERY_STATUS_DONE -(4 rows) - -DROP FUNCTION print_last_query(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; -RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_locale.out b/gpcontrib/gp_stats_collector/results/gpsc_locale.out deleted file mode 100644 index a01fe0648b9..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_locale.out +++ /dev/null @@ -1,23 +0,0 @@ --- The extension generates normalized query text and plan using jumbling functions. --- Those functions may fail when translating to wide character if the current locale --- cannot handle the character set. This test checks that even when those functions --- fail, the plan is still generated and executed. This test is partially taken from --- gp_locale. --- start_ignore -DROP DATABASE IF EXISTS gpsc_test_locale; --- end_ignore -CREATE DATABASE gpsc_test_locale WITH LC_COLLATE='C' LC_CTYPE='C' TEMPLATE=template0; -\c gpsc_test_locale -CREATE EXTENSION gp_stats_collector; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable_utility TO TRUE; -SET gpsc.enable TO TRUE; -CREATE TABLE gpsc_hi_안녕세계 (a int, 안녕세계1 text, 안녕세계2 text, 안녕세계3 text) DISTRIBUTED BY (a); -INSERT INTO gpsc_hi_안녕세계 VALUES(1, '안녕세계1 first', '안녕세2 first', '안녕세계3 first'); --- Should not see error here -UPDATE gpsc_hi_안녕세계 SET 안녕세계1='안녕세계1 first UPDATE' WHERE 안녕세계1='안녕세계1 first'; -RESET gpsc.enable; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; -DROP TABLE gpsc_hi_안녕세계; -DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_select.out b/gpcontrib/gp_stats_collector/results/gpsc_select.out deleted file mode 100644 index 3008c8f6d55..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_select.out +++ /dev/null @@ -1,136 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.enable_utility TO FALSE; --- Basic SELECT tests -SET gpsc.logging_mode to 'TBL'; -SELECT 1; - ?column? ----------- - 1 -(1 row) - -SELECT COUNT(*) FROM generate_series(1,10); - count -------- - 10 -(1 row) - -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------------------+--------------------- - -1 | SELECT 1; | QUERY_STATUS_SUBMIT - -1 | SELECT 1; | QUERY_STATUS_START - -1 | SELECT 1; | QUERY_STATUS_END - -1 | SELECT 1; | QUERY_STATUS_DONE - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END - -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Transaction test -SET gpsc.logging_mode to 'TBL'; -BEGIN; -SELECT 1; - ?column? ----------- - 1 -(1 row) - -COMMIT; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------+--------------------- - -1 | SELECT 1; | QUERY_STATUS_SUBMIT - -1 | SELECT 1; | QUERY_STATUS_START - -1 | SELECT 1; | QUERY_STATUS_END - -1 | SELECT 1; | QUERY_STATUS_DONE -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- CTE test -SET gpsc.logging_mode to 'TBL'; -WITH t AS (VALUES (1), (2)) -SELECT * FROM t; - column1 ---------- - 1 - 2 -(2 rows) - -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-----------------------------+--------------------- - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_SUBMIT - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_START - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_END - | SELECT * FROM t; | - -1 | WITH t AS (VALUES (1), (2))+| QUERY_STATUS_DONE - | SELECT * FROM t; | -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Prepared statement test -SET gpsc.logging_mode to 'TBL'; -PREPARE test_stmt AS SELECT 1; -EXECUTE test_stmt; - ?column? ----------- - 1 -(1 row) - -DEALLOCATE test_stmt; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------------+--------------------- - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_SUBMIT - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_START - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_END - -1 | PREPARE test_stmt AS SELECT 1; | QUERY_STATUS_DONE -(4 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_uds.out b/gpcontrib/gp_stats_collector/results/gpsc_uds.out deleted file mode 100644 index e8bca79e669..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_uds.out +++ /dev/null @@ -1,42 +0,0 @@ --- Test UDS socket --- start_ignore -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; --- end_ignore -\set UDS_PATH '/tmp/gpsc_test.sock' --- Configure extension to send via UDS -SET gpsc.uds_path TO :'UDS_PATH'; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.logging_mode TO 'UDS'; --- Start receiver -SELECT gpsc.__test_uds_start_server(:'UDS_PATH'); - __test_uds_start_server -------------------------- -(0 rows) - --- Send -SELECT 1; - ?column? ----------- - 1 -(1 row) - --- Receive -SELECT gpsc.__test_uds_receive() > 0 as received; - received ----------- - t -(1 row) - --- Stop receiver -SELECT gpsc.__test_uds_stop_server(); - __test_uds_stop_server ------------------------- -(0 rows) - --- Cleanup -DROP EXTENSION gp_stats_collector; -RESET gpsc.uds_path; -RESET gpsc.ignored_users_list; -RESET gpsc.enable; -RESET gpsc.logging_mode; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out b/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out deleted file mode 100644 index db3949f3152..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_utf8_trim.out +++ /dev/null @@ -1,68 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS gp_stats_collector; -CREATE OR REPLACE FUNCTION get_marked_query(marker TEXT) -RETURNS TEXT AS $$ - SELECT query_text - FROM gpsc.log - WHERE query_text LIKE '%' || marker || '%' - ORDER BY datetime DESC - LIMIT 1 -$$ LANGUAGE sql VOLATILE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; --- Test 1: 1 byte chars -SET gpsc.max_text_size to 19; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test1*/ 'HelloWorld'; - ?column? ------------- - HelloWorld -(1 row) - -RESET gpsc.logging_mode; -SELECT octet_length(get_marked_query('test1')) = 19 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Test 2: 2 byte chars -SET gpsc.max_text_size to 19; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test2*/ 'РУССКИЙЯЗЫК'; - ?column? -------------- - РУССКИЙЯЗЫК -(1 row) - -RESET gpsc.logging_mode; --- Character 'Р' has two bytes and cut in the middle => not included. -SELECT octet_length(get_marked_query('test2')) = 18 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Test 3: 4 byte chars -SET gpsc.max_text_size to 21; -SET gpsc.logging_mode to 'TBL'; -SELECT /*test3*/ '😀'; - ?column? ----------- - 😀 -(1 row) - -RESET gpsc.logging_mode; --- Emoji has 4 bytes and cut before the last byte => not included. -SELECT octet_length(get_marked_query('test3')) = 18 AS correct_length; - correct_length ----------------- - t -(1 row) - --- Cleanup -DROP FUNCTION get_marked_query(TEXT); -RESET gpsc.max_text_size; -RESET gpsc.logging_mode; -RESET gpsc.enable; -RESET gpsc.ignored_users_list; -DROP EXTENSION gp_stats_collector; diff --git a/gpcontrib/gp_stats_collector/results/gpsc_utility.out b/gpcontrib/gp_stats_collector/results/gpsc_utility.out deleted file mode 100644 index e8e28614370..00000000000 --- a/gpcontrib/gp_stats_collector/results/gpsc_utility.out +++ /dev/null @@ -1,248 +0,0 @@ -CREATE EXTENSION gp_stats_collector; -CREATE OR REPLACE FUNCTION gpsc_status_order(status text) -RETURNS integer -AS $$ -BEGIN - RETURN CASE status - WHEN 'QUERY_STATUS_SUBMIT' THEN 1 - WHEN 'QUERY_STATUS_START' THEN 2 - WHEN 'QUERY_STATUS_END' THEN 3 - WHEN 'QUERY_STATUS_DONE' THEN 4 - ELSE 999 - END; -END; -$$ LANGUAGE plpgsql IMMUTABLE; -SET gpsc.ignored_users_list TO ''; -SET gpsc.enable TO TRUE; -SET gpsc.enable_utility TO TRUE; -SET gpsc.report_nested_queries TO TRUE; -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE test_table (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -CREATE INDEX test_idx ON test_table(a); -ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; -DROP TABLE test_table; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------------------------------+--------------------- - -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE test_table (a int, b text); | QUERY_STATUS_DONE - -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_SUBMIT - -1 | CREATE INDEX test_idx ON test_table(a); | QUERY_STATUS_DONE - -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_SUBMIT - -1 | ALTER TABLE test_table ADD COLUMN c int DEFAULT 1; | QUERY_STATUS_DONE - -1 | DROP TABLE test_table; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE test_table; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Partitioning -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE pt_test (a int, b int) -DISTRIBUTED BY (a) -PARTITION BY RANGE (a) -(START (0) END (100) EVERY (50)); -DROP TABLE pt_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------+--------------------- - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_SUBMIT - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | CREATE TABLE pt_test (a int, b int)+| QUERY_STATUS_DONE - | DISTRIBUTED BY (a) +| - | PARTITION BY RANGE (a) +| - | (START (0) END (100) EVERY (50)); | - -1 | DROP TABLE pt_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE pt_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Views and Functions -SET gpsc.logging_mode to 'TBL'; -CREATE VIEW test_view AS SELECT 1 AS a; -CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; -DROP VIEW test_view; -DROP FUNCTION test_func(int); -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------------------------------------------------------------------------------+--------------------- - -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_SUBMIT - -1 | CREATE VIEW test_view AS SELECT 1 AS a; | QUERY_STATUS_DONE - -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_SUBMIT - -1 | CREATE FUNCTION test_func(i int) RETURNS int AS $$ SELECT $1 + 1; $$ LANGUAGE SQL; | QUERY_STATUS_DONE - -1 | DROP VIEW test_view; | QUERY_STATUS_SUBMIT - -1 | DROP VIEW test_view; | QUERY_STATUS_DONE - -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_SUBMIT - -1 | DROP FUNCTION test_func(int); | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(10 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Transaction Operations -SET gpsc.logging_mode to 'TBL'; -BEGIN; -SAVEPOINT sp1; -ROLLBACK TO sp1; -COMMIT; -BEGIN; -SAVEPOINT sp2; -ABORT; -BEGIN; -ROLLBACK; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+--------------------------+--------------------- - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK TO sp1; | QUERY_STATUS_DONE - -1 | COMMIT; | QUERY_STATUS_SUBMIT - -1 | COMMIT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | SAVEPOINT sp2; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_SUBMIT - -1 | ABORT; | QUERY_STATUS_DONE - -1 | BEGIN; | QUERY_STATUS_SUBMIT - -1 | BEGIN; | QUERY_STATUS_DONE - -1 | ROLLBACK; | QUERY_STATUS_SUBMIT - -1 | ROLLBACK; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(18 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- DML Operations -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE dml_test (a int, b text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO dml_test VALUES (1, 'test'); -UPDATE dml_test SET b = 'updated' WHERE a = 1; -DELETE FROM dml_test WHERE a = 1; -DROP TABLE dml_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+----------------------------------------+--------------------- - -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE dml_test (a int, b text); | QUERY_STATUS_DONE - -1 | DROP TABLE dml_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE dml_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- COPY Operations -SET gpsc.logging_mode to 'TBL'; -CREATE TABLE copy_test (a int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -COPY (SELECT 1) TO STDOUT; -1 -DROP TABLE copy_test; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+---------------------------------+--------------------- - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_SUBMIT - -1 | CREATE TABLE copy_test (a int); | QUERY_STATUS_DONE - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_SUBMIT - -1 | COPY (SELECT 1) TO STDOUT; | QUERY_STATUS_DONE - -1 | DROP TABLE copy_test; | QUERY_STATUS_SUBMIT - -1 | DROP TABLE copy_test; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- Prepared Statements and error during execute -SET gpsc.logging_mode to 'TBL'; -PREPARE test_prep(int) AS SELECT $1/0 AS value; -EXECUTE test_prep(0::int); -ERROR: division by zero -DEALLOCATE test_prep; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+-------------------------------------------------+--------------------- - -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_SUBMIT - -1 | PREPARE test_prep(int) AS SELECT $1/0 AS value; | QUERY_STATUS_DONE - -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_SUBMIT - -1 | EXECUTE test_prep(0::int); | QUERY_STATUS_ERROR - -1 | DEALLOCATE test_prep; | QUERY_STATUS_SUBMIT - -1 | DEALLOCATE test_prep; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(8 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - --- GUC Settings -SET gpsc.logging_mode to 'TBL'; -SET gpsc.report_nested_queries TO FALSE; -RESET gpsc.report_nested_queries; -RESET gpsc.logging_mode; -SELECT segid, query_text, query_status FROM gpsc.log WHERE segid = -1 AND utility = true ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; - segid | query_text | query_status --------+------------------------------------------+--------------------- - -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_SUBMIT - -1 | SET gpsc.report_nested_queries TO FALSE; | QUERY_STATUS_DONE - -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.report_nested_queries; | QUERY_STATUS_DONE - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_SUBMIT - -1 | RESET gpsc.logging_mode; | QUERY_STATUS_DONE -(6 rows) - -SELECT gpsc.truncate_log() IS NOT NULL AS t; - t ---- -(0 rows) - -DROP FUNCTION gpsc_status_order(text); -DROP EXTENSION gp_stats_collector; -RESET gpsc.enable; -RESET gpsc.report_nested_queries; -RESET gpsc.enable_utility; -RESET gpsc.ignored_users_list; diff --git a/gpcontrib/gp_stats_collector/src/Config.cpp b/gpcontrib/gp_stats_collector/src/Config.cpp index 2f40b30e922..08a8d8cff86 100644 --- a/gpcontrib/gp_stats_collector/src/Config.cpp +++ b/gpcontrib/gp_stats_collector/src/Config.cpp @@ -26,11 +26,11 @@ */ #include "Config.h" -#include "memory/gpdbwrappers.h" #include #include #include #include +#include "memory/gpdbwrappers.h" extern "C" { #include "postgres.h" @@ -43,135 +43,149 @@ static bool guc_enable_cdbstats = true; static bool guc_enable_collector = false; static bool guc_report_nested_queries = true; static char *guc_ignored_users = nullptr; -static int guc_max_text_size = 1 << 20; // in bytes (1MB) -static int guc_max_plan_size = 1024; // in KB -static int guc_min_analyze_time = 10000; // in ms +static int guc_max_text_size = 1 << 20; // in bytes (1MB) +static int guc_max_plan_size = 1024; // in KB +static int guc_min_analyze_time = 10000; // in ms static int guc_logging_mode = LOG_MODE_UDS; static bool guc_enable_utility = false; static const struct config_enum_entry logging_mode_options[] = { - {"uds", LOG_MODE_UDS, false /* hidden */}, - {"tbl", LOG_MODE_TBL, false}, - {NULL, 0, false}}; + {"uds", LOG_MODE_UDS, false /* hidden */}, + {"tbl", LOG_MODE_TBL, false}, + {NULL, 0, false}}; static bool ignored_users_guc_dirty = false; -static void assign_ignored_users_hook(const char *, void *) { - ignored_users_guc_dirty = true; +static void +assign_ignored_users_hook(const char *, void *) +{ + ignored_users_guc_dirty = true; } -void Config::init_gucs() { - DefineCustomStringVariable( - "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, - &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable", "Enable metrics collector", 0LL, &guc_enable_collector, - false, PGC_SUSET, GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, - &guc_enable_analyze, true, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, - &guc_enable_cdbstats, true, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomBoolVariable( - "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, - &guc_report_nested_queries, true, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); - - DefineCustomStringVariable("gpsc.ignored_users_list", - "Make gpsc ignore queries issued by given users", - 0LL, &guc_ignored_users, - "", PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, - assign_ignored_users_hook, 0LL); - - DefineCustomIntVariable( - "gpsc.max_text_size", - "Make gpsc trim query texts longer than configured size in bytes", NULL, - &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); - - DefineCustomIntVariable( - "gpsc.max_plan_size", - "Make gpsc trim plan longer than configured size", NULL, - &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); - - DefineCustomIntVariable( - "gpsc.min_analyze_time", - "Sets the minimum execution time above which plans will be logged.", - "Zero prints all plans. -1 turns this feature off.", - &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); - - DefineCustomEnumVariable( - "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, - &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, - NULL); - - DefineCustomBoolVariable( - "gpsc.enable_utility", "Collect utility statement stats", NULL, - &guc_enable_utility, false, PGC_USERSET, - GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); +void +Config::init_gucs() +{ + DefineCustomStringVariable( + "gpsc.uds_path", "Sets filesystem path of the agent socket", 0LL, + &guc_uds_path, "/tmp/gpsc_agent.sock", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable("gpsc.enable", "Enable metrics collector", 0LL, + &guc_enable_collector, false, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, + 0LL); + + DefineCustomBoolVariable( + "gpsc.enable_analyze", "Collect analyze metrics in gpsc", 0LL, + &guc_enable_analyze, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "gpsc.enable_cdbstats", "Collect CDB metrics in gpsc", 0LL, + &guc_enable_cdbstats, true, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomBoolVariable( + "gpsc.report_nested_queries", "Collect stats on nested queries", 0LL, + &guc_report_nested_queries, true, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, 0LL, 0LL); + + DefineCustomStringVariable("gpsc.ignored_users_list", + "Make gpsc ignore queries issued by given users", + 0LL, &guc_ignored_users, "", PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, 0LL, + assign_ignored_users_hook, 0LL); + + DefineCustomIntVariable( + "gpsc.max_text_size", + "Make gpsc trim query texts longer than configured size in bytes", NULL, + &guc_max_text_size, 1 << 20 /* 1MB */, 0, INT_MAX, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); + + DefineCustomIntVariable( + "gpsc.max_plan_size", "Make gpsc trim plan longer than configured size", + NULL, &guc_max_plan_size, 1024, 0, INT_MAX / 1024, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_KB, NULL, NULL, NULL); + + DefineCustomIntVariable( + "gpsc.min_analyze_time", + "Sets the minimum execution time above which plans will be logged.", + "Zero prints all plans. -1 turns this feature off.", + &guc_min_analyze_time, 10000, -1, INT_MAX, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_UNIT_MS, NULL, NULL, NULL); + + DefineCustomEnumVariable( + "gpsc.logging_mode", "Logging mode: UDS or PG Table", NULL, + &guc_logging_mode, LOG_MODE_UDS, logging_mode_options, PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC | GUC_SUPERUSER_ONLY, NULL, NULL, + NULL); + + DefineCustomBoolVariable( + "gpsc.enable_utility", "Collect utility statement stats", NULL, + &guc_enable_utility, false, PGC_USERSET, + GUC_NOT_IN_SAMPLE | GUC_GPDB_NEED_SYNC, NULL, NULL, NULL); } -void Config::update_ignored_users(const char *new_guc_ignored_users) { - auto new_ignored_users_set = std::make_unique(); - if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') { - /* Need a modifiable copy of string */ - char *rawstring = gpdb::pstrdup(new_guc_ignored_users); - List *elemlist; - ListCell *l; - - /* Parse string into list of identifiers */ - if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) { - /* syntax error in list */ - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); - ereport( - LOG, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg( - "invalid list syntax in parameter gpsc.ignored_users_list"))); - return; - } - foreach (l, elemlist) { - new_ignored_users_set->insert((char *)lfirst(l)); - } - gpdb::pfree(rawstring); - gpdb::list_free(elemlist); - } - ignored_users_ = std::move(new_ignored_users_set); +void +Config::update_ignored_users(const char *new_guc_ignored_users) +{ + auto new_ignored_users_set = std::make_unique(); + if (new_guc_ignored_users != nullptr && new_guc_ignored_users[0] != '\0') + { + /* Need a modifiable copy of string */ + char *rawstring = gpdb::pstrdup(new_guc_ignored_users); + List *elemlist; + ListCell *l; + + /* Parse string into list of identifiers */ + if (!gpdb::split_identifier_string(rawstring, ',', &elemlist)) + { + /* syntax error in list */ + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); + ereport( + LOG, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg( + "invalid list syntax in parameter gpsc.ignored_users_list"))); + return; + } + foreach (l, elemlist) + { + new_ignored_users_set->insert((char *) lfirst(l)); + } + gpdb::pfree(rawstring); + gpdb::list_free(elemlist); + } + ignored_users_ = std::move(new_ignored_users_set); } -bool Config::filter_user(const std::string &username) const { - if (!ignored_users_) { - return true; - } - return ignored_users_->find(username) != ignored_users_->end(); +bool +Config::filter_user(const std::string &username) const +{ + if (!ignored_users_) + { + return true; + } + return ignored_users_->find(username) != ignored_users_->end(); } -void Config::sync() { - if (ignored_users_guc_dirty) { - update_ignored_users(guc_ignored_users); - ignored_users_guc_dirty = false; - } - uds_path_ = guc_uds_path; - enable_analyze_ = guc_enable_analyze; - enable_cdbstats_ = guc_enable_cdbstats; - enable_collector_ = guc_enable_collector; - enable_utility_ = guc_enable_utility; - report_nested_queries_ = guc_report_nested_queries; - max_text_size_ = guc_max_text_size; - max_plan_size_ = guc_max_plan_size; - min_analyze_time_ = guc_min_analyze_time; - logging_mode_ = guc_logging_mode; +void +Config::sync() +{ + if (ignored_users_guc_dirty) + { + update_ignored_users(guc_ignored_users); + ignored_users_guc_dirty = false; + } + uds_path_ = guc_uds_path; + enable_analyze_ = guc_enable_analyze; + enable_cdbstats_ = guc_enable_cdbstats; + enable_collector_ = guc_enable_collector; + enable_utility_ = guc_enable_utility; + report_nested_queries_ = guc_report_nested_queries; + max_text_size_ = guc_max_text_size; + max_plan_size_ = guc_max_plan_size; + min_analyze_time_ = guc_min_analyze_time; + logging_mode_ = guc_logging_mode; } diff --git a/gpcontrib/gp_stats_collector/src/Config.h b/gpcontrib/gp_stats_collector/src/Config.h index 91a1ffe44f2..259799e5135 100644 --- a/gpcontrib/gp_stats_collector/src/Config.h +++ b/gpcontrib/gp_stats_collector/src/Config.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef CONFIG_H +#define CONFIG_H #include #include @@ -36,36 +37,79 @@ using IgnoredUsers = std::unordered_set; -class Config { +class Config +{ public: - static void init_gucs(); + static void init_gucs(); - void sync(); + void sync(); - const std::string &uds_path() const { return uds_path_; } - bool enable_analyze() const { return enable_analyze_; } - bool enable_cdbstats() const { return enable_cdbstats_; } - bool enable_collector() const { return enable_collector_; } - bool enable_utility() const { return enable_utility_; } - bool report_nested_queries() const { return report_nested_queries_; } - int max_text_size() const { return max_text_size_; } - int max_plan_size() const { return max_plan_size_ * 1024; } - int min_analyze_time() const { return min_analyze_time_; } - int logging_mode() const { return logging_mode_; } - bool filter_user(const std::string &username) const; + const std::string & + uds_path() const + { + return uds_path_; + } + bool + enable_analyze() const + { + return enable_analyze_; + } + bool + enable_cdbstats() const + { + return enable_cdbstats_; + } + bool + enable_collector() const + { + return enable_collector_; + } + bool + enable_utility() const + { + return enable_utility_; + } + bool + report_nested_queries() const + { + return report_nested_queries_; + } + int + max_text_size() const + { + return max_text_size_; + } + int + max_plan_size() const + { + return max_plan_size_ * 1024; + } + int + min_analyze_time() const + { + return min_analyze_time_; + } + int + logging_mode() const + { + return logging_mode_; + } + bool filter_user(const std::string &username) const; private: - void update_ignored_users(const char *new_guc_ignored_users); + void update_ignored_users(const char *new_guc_ignored_users); - std::unique_ptr ignored_users_; - std::string uds_path_; - bool enable_analyze_; - bool enable_cdbstats_; - bool enable_collector_; - bool enable_utility_; - bool report_nested_queries_; - int max_text_size_; - int max_plan_size_; - int min_analyze_time_; - int logging_mode_; + std::unique_ptr ignored_users_; + std::string uds_path_; + bool enable_analyze_; + bool enable_cdbstats_; + bool enable_collector_; + bool enable_utility_; + bool report_nested_queries_; + int max_text_size_; + int max_plan_size_; + int min_analyze_time_; + int logging_mode_; }; + +#endif /* CONFIG_H */ diff --git a/gpcontrib/gp_stats_collector/src/EventSender.cpp b/gpcontrib/gp_stats_collector/src/EventSender.cpp index c0faaf0ad0e..0bc44c1198d 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.cpp +++ b/gpcontrib/gp_stats_collector/src/EventSender.cpp @@ -26,8 +26,8 @@ */ #include "UDSConnector.h" -#include "memory/gpdbwrappers.h" #include "log/LogOps.h" +#include "memory/gpdbwrappers.h" #define typeid __typeid extern "C" { @@ -47,487 +47,599 @@ extern "C" { #include "PgUtils.h" #include "ProtoUtils.h" -#define need_collect_analyze() \ - (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ - config.enable_analyze()) - -bool EventSender::verify_query(QueryDesc *query_desc, QueryState state, - bool utility) { - if (!proto_verified) { - return false; - } - if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) { - return false; - } - - switch (state) { - case QueryState::SUBMIT: - // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent - // states. Without caching, a query that unsets/sets filtering GUCs would - // see different filter criteria at DONE, because at SUBMIT the query was - // not executed yet, causing DONE to be skipped/added. - config.sync(); - - if (!config.enable_collector()) { - return false; - } - - if (utility && !config.enable_utility()) { - return false; - } - - // Register qkey for a nested query we won't report, - // so we can detect nesting_level > 0 and skip reporting at end/done. - if (!need_report_nested_query() && nesting_level > 0) { - QueryKey::register_qkey(query_desc, nesting_level); - return false; - } - if (is_top_level_query(query_desc, nesting_level)) { - nested_timing = 0; - nested_calls = 0; - } - break; - case QueryState::START: - if (!qdesc_submitted(query_desc)) { - collect_query_submit(query_desc, false /* utility */); - } - break; - case QueryState::DONE: - if (utility && !config.enable_utility()) { - return false; - } - default: - break; - } - - if (filter_query(query_desc)) { - return false; - } - if (!nesting_is_valid(query_desc, nesting_level)) { - return false; - } - - return true; +#define need_collect_analyze() \ + (Gp_role == GP_ROLE_DISPATCH && config.min_analyze_time() >= 0 && \ + config.enable_analyze()) + +bool +EventSender::verify_query(QueryDesc *query_desc, QueryState state, bool utility) +{ + if (!proto_verified) + { + return false; + } + if (Gp_role != GP_ROLE_DISPATCH && Gp_role != GP_ROLE_EXECUTE) + { + return false; + } + + switch (state) + { + case QueryState::SUBMIT: + // Cache GUCs once at SUBMIT. Synced GUCs are visible to all subsequent + // states. Without caching, a query that unsets/sets filtering GUCs would + // see different filter criteria at DONE, because at SUBMIT the query was + // not executed yet, causing DONE to be skipped/added. + config.sync(); + + if (!config.enable_collector()) + { + return false; + } + + if (utility && !config.enable_utility()) + { + return false; + } + + // Register qkey for a nested query we won't report, + // so we can detect nesting_level > 0 and skip reporting at end/done. + if (!need_report_nested_query() && nesting_level > 0) + { + QueryKey::register_qkey(query_desc, nesting_level); + return false; + } + if (is_top_level_query(query_desc, nesting_level)) + { + nested_timing = 0; + nested_calls = 0; + } + break; + case QueryState::START: + if (!qdesc_submitted(query_desc)) + { + collect_query_submit(query_desc, false /* utility */); + } + break; + case QueryState::DONE: + if (utility && !config.enable_utility()) + { + return false; + } + default: + break; + } + + if (filter_query(query_desc)) + { + return false; + } + if (!nesting_is_valid(query_desc, nesting_level)) + { + return false; + } + + return true; } -bool EventSender::log_query_req(const gpsc::SetQueryReq &req, - const std::string &event, bool utility) { - bool clear_big_fields = false; - switch (config.logging_mode()) { - case LOG_MODE_UDS: - clear_big_fields = UDSConnector::report_query(req, event, config); - break; - case LOG_MODE_TBL: - gpdb::insert_log(req, utility); - clear_big_fields = false; - break; - default: - Assert(false); - } - return clear_big_fields; +bool +EventSender::log_query_req(const gpsc::SetQueryReq &req, + const std::string &event, bool utility) +{ + bool clear_big_fields = false; + switch (config.logging_mode()) + { + case LOG_MODE_UDS: + clear_big_fields = UDSConnector::report_query(req, event, config); + break; + case LOG_MODE_TBL: + gpdb::insert_log(req, utility); + clear_big_fields = false; + break; + default: + Assert(false); + } + return clear_big_fields; } -void EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, - bool utility, ErrorData *edata) { - auto *query_desc = reinterpret_cast(arg); - switch (status) { - case METRICS_PLAN_NODE_INITIALIZE: - case METRICS_PLAN_NODE_EXECUTING: - case METRICS_PLAN_NODE_FINISHED: - // TODO - break; - case METRICS_QUERY_SUBMIT: - collect_query_submit(query_desc, utility); - break; - case METRICS_QUERY_START: - // no-op: executor_after_start is enough - break; - case METRICS_QUERY_CANCELING: - // it appears we're only interested in the actual CANCELED event. - // for now we will ignore CANCELING state unless otherwise requested from - // end users - break; - case METRICS_QUERY_DONE: - case METRICS_QUERY_ERROR: - case METRICS_QUERY_CANCELED: - case METRICS_INNER_QUERY_DONE: - collect_query_done(query_desc, utility, status, edata); - break; - default: - ereport(FATAL, (errmsg("Unknown query status: %d", status))); - } +void +EventSender::query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata) +{ + auto *query_desc = reinterpret_cast(arg); + switch (status) + { + case METRICS_PLAN_NODE_INITIALIZE: + case METRICS_PLAN_NODE_EXECUTING: + case METRICS_PLAN_NODE_FINISHED: + // TODO + break; + case METRICS_QUERY_SUBMIT: + collect_query_submit(query_desc, utility); + break; + case METRICS_QUERY_START: + // no-op: executor_after_start is enough + break; + case METRICS_QUERY_CANCELING: + // it appears we're only interested in the actual CANCELED event. + // for now we will ignore CANCELING state unless otherwise requested from + // end users + break; + case METRICS_QUERY_DONE: + case METRICS_QUERY_ERROR: + case METRICS_QUERY_CANCELED: + case METRICS_INNER_QUERY_DONE: + collect_query_done(query_desc, utility, status, edata); + break; + default: + ereport(ERROR, (errmsg("Unknown query status: %d", status))); + } } -void EventSender::executor_before_start(QueryDesc *query_desc, int eflags) { - if (!verify_query(query_desc, QueryState::START, false /* utility*/)) { - return; - } - - if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && - (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) { - query_desc->instrument_options |= INSTRUMENT_BUFFERS; - query_desc->instrument_options |= INSTRUMENT_ROWS; - query_desc->instrument_options |= INSTRUMENT_TIMER; - if (config.enable_cdbstats()) { - query_desc->instrument_options |= INSTRUMENT_CDB; - if (!query_desc->showstatctx) { - instr_time starttime; - INSTR_TIME_SET_CURRENT(starttime); - query_desc->showstatctx = - gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); - } - } - } +void +EventSender::executor_before_start(QueryDesc *query_desc, int eflags) +{ + if (!verify_query(query_desc, QueryState::START, false /* utility*/)) + { + return; + } + + if (Gp_role == GP_ROLE_DISPATCH && config.enable_analyze() && + (eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + { + query_desc->instrument_options |= INSTRUMENT_BUFFERS; + query_desc->instrument_options |= INSTRUMENT_ROWS; + query_desc->instrument_options |= INSTRUMENT_TIMER; + if (config.enable_cdbstats()) + { + query_desc->instrument_options |= INSTRUMENT_CDB; + if (!query_desc->showstatctx) + { + instr_time starttime; + INSTR_TIME_SET_CURRENT(starttime); + query_desc->showstatctx = + gpdb::cdbexplain_showExecStatsBegin(query_desc, starttime); + } + } + } } -void EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) { - if (!verify_query(query_desc, QueryState::START, false /* utility */)) { - return; - } - - auto &query = get_query(query_desc); - auto query_msg = query.message.get(); - *query_msg->mutable_start_time() = current_ts(); - update_query_state(query, QueryState::START, false /* utility */); - set_query_plan(query_msg, query_desc, config); - if (need_collect_analyze()) { - // Set up to track total elapsed time during query run. - // Make sure the space is allocated in the per-query - // context so it will go away at executor_end. - if (query_desc->totaltime == NULL) { - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); - gpdb::mem_ctx_switch_to(oldcxt); - } - } - gpsc::GPMetrics stats; - std::swap(stats, *query_msg->mutable_query_metrics()); - if (log_query_req(*query_msg, "started", false /* utility */)) { - clear_big_fields(query_msg); - } - std::swap(stats, *query_msg->mutable_query_metrics()); +void +EventSender::executor_after_start(QueryDesc *query_desc, int /* eflags*/) +{ + if (!verify_query(query_desc, QueryState::START, false /* utility */)) + { + return; + } + + auto &query = get_query(query_desc); + auto query_msg = query.message.get(); + *query_msg->mutable_start_time() = current_ts(); + update_query_state(query, QueryState::START, false /* utility */); + set_query_plan(query_msg, query_desc, config); + if (need_collect_analyze()) + { + // Set up to track total elapsed time during query run. + // Make sure the space is allocated in the per-query + // context so it will go away at executor_end. + if (query_desc->totaltime == NULL) + { + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + query_desc->totaltime = gpdb::instr_alloc(1, INSTRUMENT_ALL, false); + gpdb::mem_ctx_switch_to(oldcxt); + } + } + gpsc::GPMetrics stats; + std::swap(stats, *query_msg->mutable_query_metrics()); + if (log_query_req(*query_msg, "started", false /* utility */)) + { + clear_big_fields(query_msg); + } + std::swap(stats, *query_msg->mutable_query_metrics()); } -void EventSender::executor_end(QueryDesc *query_desc) { - if (!verify_query(query_desc, QueryState::END, false /* utility */)) { - return; - } - - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg->mutable_end_time() = current_ts(); - update_query_state(query, QueryState::END, false /* utility */); - if (is_top_level_query(query_desc, nesting_level)) { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, - nested_timing); - } else { - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); - } - if (log_query_req(*query_msg, "ended", false /* utility */)) { - clear_big_fields(query_msg); - } +void +EventSender::executor_end(QueryDesc *query_desc) +{ + if (!verify_query(query_desc, QueryState::END, false /* utility */)) + { + return; + } + + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg->mutable_end_time() = current_ts(); + update_query_state(query, QueryState::END, false /* utility */); + if (is_top_level_query(query_desc, nesting_level)) + { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } + else + { + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); + } + if (log_query_req(*query_msg, "ended", false /* utility */)) + { + clear_big_fields(query_msg); + } } -void EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) { - if (!verify_query(query_desc, QueryState::SUBMIT, utility)) { - return; - } - - submit_query(query_desc); - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); - *query_msg->mutable_submit_time() = current_ts(); - set_query_info(query_msg); - set_qi_nesting_level(query_msg, nesting_level); - set_qi_slice_id(query_msg); - set_query_text(query_msg, query_desc, config); - if (log_query_req(*query_msg, "submit", utility)) { - clear_big_fields(query_msg); - } - // take initial metrics snapshot so that we can safely take diff afterwards - // in END or DONE events. - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); +void +EventSender::collect_query_submit(QueryDesc *query_desc, bool utility) +{ + if (!verify_query(query_desc, QueryState::SUBMIT, utility)) + { + return; + } + + submit_query(query_desc); + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + *query_msg = create_query_req(gpsc::QueryStatus::QUERY_STATUS_SUBMIT); + *query_msg->mutable_submit_time() = current_ts(); + set_query_info(query_msg); + set_qi_nesting_level(query_msg, nesting_level); + set_qi_slice_id(query_msg); + set_query_text(query_msg, query_desc, config); + if (log_query_req(*query_msg, "submit", utility)) + { + clear_big_fields(query_msg); + } + // take initial metrics snapshot so that we can safely take diff afterwards + // in END or DONE events. + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, 0, 0); #ifdef IC_TEARDOWN_HOOK - // same for interconnect statistics - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + // same for interconnect statistics + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif } -void EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status, bool utility, - ErrorData *edata) { - gpsc::QueryStatus query_status; - std::string msg; - switch (status) { - case METRICS_QUERY_DONE: - case METRICS_INNER_QUERY_DONE: - query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; - msg = "done"; - break; - case METRICS_QUERY_ERROR: - query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; - msg = "error"; - break; - case METRICS_QUERY_CANCELING: - // at the moment we don't track this event, but I`ll leave this code - // here just in case - Assert(false); - query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; - msg = "cancelling"; - break; - case METRICS_QUERY_CANCELED: - query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; - msg = "cancelled"; - break; - default: - ereport(FATAL, - (errmsg("Unexpected query status in query_done hook: %d", status))); - } - auto prev_state = query.state; - update_query_state(query, QueryState::DONE, utility, - query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); - auto query_msg = query.message.get(); - query_msg->set_query_status(query_status); - if (status == METRICS_QUERY_ERROR) { - bool error_flushed = elog_message() == NULL; - if (error_flushed && (edata == NULL || edata->message == NULL)) { - ereport(WARNING, (errmsg("GPSC missing error message"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } else { - set_qi_error_message( - query_msg, error_flushed ? edata->message : elog_message(), config); - } - } - if (prev_state == START) { - // We've missed ExecutorEnd call due to query cancel or error. It's - // fine, but now we need to collect and report execution stats - *query_msg->mutable_end_time() = current_ts(); - set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, nested_calls, - nested_timing); - } +void +EventSender::report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status, bool utility, + ErrorData *edata) +{ + gpsc::QueryStatus query_status; + std::string msg; + switch (status) + { + case METRICS_QUERY_DONE: + case METRICS_INNER_QUERY_DONE: + query_status = gpsc::QueryStatus::QUERY_STATUS_DONE; + msg = "done"; + break; + case METRICS_QUERY_ERROR: + query_status = gpsc::QueryStatus::QUERY_STATUS_ERROR; + msg = "error"; + break; + case METRICS_QUERY_CANCELING: + // at the moment we don't track this event, but I`ll leave this code + // here just in case + Assert(false); + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELLING; + msg = "cancelling"; + break; + case METRICS_QUERY_CANCELED: + query_status = gpsc::QueryStatus::QUERY_STATUS_CANCELED; + msg = "cancelled"; + break; + default: + ereport(ERROR, + (errmsg("Unexpected query status in query_done hook: %d", + status))); + } + auto prev_state = query.state; + update_query_state(query, QueryState::DONE, utility, + query_status == gpsc::QueryStatus::QUERY_STATUS_DONE); + auto query_msg = query.message.get(); + query_msg->set_query_status(query_status); + if (status == METRICS_QUERY_ERROR) + { + bool error_flushed = elog_message() == NULL; + if (error_flushed && (edata == NULL || edata->message == NULL)) + { + ereport(WARNING, (errmsg("GPSC missing error message"))); + ereport(DEBUG3, (errmsg("GPSC query sourceText: %s", + query_desc->sourceText))); + } + else + { + set_qi_error_message( + query_msg, error_flushed ? edata->message : elog_message(), + config); + } + } + if (prev_state == START) + { + // We've missed ExecutorEnd call due to query cancel or error. It's + // fine, but now we need to collect and report execution stats + *query_msg->mutable_end_time() = current_ts(); + set_gp_metrics(query_msg->mutable_query_metrics(), query_desc, + nested_calls, nested_timing); + } #ifdef IC_TEARDOWN_HOOK - ic_metrics_collect(); - set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), - &ic_statistics); + ic_metrics_collect(); + set_ic_stats(query_msg->mutable_query_metrics()->mutable_instrumentation(), + &ic_statistics); #endif - (void)log_query_req(*query_msg, msg, utility); + (void) log_query_req(*query_msg, msg, utility); } -void EventSender::collect_query_done(QueryDesc *query_desc, bool utility, - QueryMetricsStatus status, - ErrorData *edata) { - if (!verify_query(query_desc, QueryState::DONE, utility)) { - return; - } - - // Skip sending done message if query errored before submit. - if (!qdesc_submitted(query_desc)) { - if (status != METRICS_QUERY_ERROR) { - ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " - "unsubmitted and unerrored query"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } - return; - } - - if (queries.empty()) { - ereport(WARNING, (errmsg("GPSC cannot find query to process DONE hook"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - return; - } - auto &query = get_query(query_desc); - - report_query_done(query_desc, query, status, utility, edata); - - if (need_report_nested_query()) - update_nested_counters(query_desc); - - queries.erase(QueryKey::from_qdesc(query_desc)); - pfree(query_desc->gpsc_query_key); - query_desc->gpsc_query_key = NULL; +void +EventSender::collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata) +{ + if (!verify_query(query_desc, QueryState::DONE, utility)) + { + return; + } + + // Skip sending done message if query errored before submit. + if (!qdesc_submitted(query_desc)) + { + if (status != METRICS_QUERY_ERROR) + { + ereport(WARNING, (errmsg("GPSC trying to process DONE hook for " + "unsubmitted and unerrored query"))); + ereport(DEBUG3, (errmsg("GPSC query sourceText: %s", + query_desc->sourceText))); + } + return; + } + + if (queries.empty()) + { + ereport(WARNING, + (errmsg("GPSC cannot find query to process DONE hook"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + return; + } + auto &query = get_query(query_desc); + + report_query_done(query_desc, query, status, utility, edata); + + if (need_report_nested_query()) + update_nested_counters(query_desc); + + queries.erase(QueryKey::from_qdesc(query_desc)); + pfree(query_desc->gpsc_query_key); + query_desc->gpsc_query_key = NULL; } -void EventSender::ic_metrics_collect() { +void +EventSender::ic_metrics_collect() +{ #ifdef IC_TEARDOWN_HOOK - if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) { - return; - } - if (!proto_verified || gp_command_count == 0 || !config.enable_collector() || - config.filter_user(get_user_name())) { - return; - } - // we also would like to know nesting level here and filter queries BUT we - // don't have this kind of information from this callback. Will have to - // collect stats anyways and throw it away later, if necessary - auto metrics = UDPIFCGetICStats(); - ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; - ic_statistics.recvQueueSizeCountingTime += metrics.recvQueueSizeCountingTime; - ic_statistics.totalCapacity += metrics.totalCapacity; - ic_statistics.capacityCountingTime += metrics.capacityCountingTime; - ic_statistics.totalBuffers += metrics.totalBuffers; - ic_statistics.bufferCountingTime += metrics.bufferCountingTime; - ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; - ic_statistics.retransmits += metrics.retransmits; - ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; - ic_statistics.mismatchNum += metrics.mismatchNum; - ic_statistics.crcErrors += metrics.crcErrors; - ic_statistics.sndPktNum += metrics.sndPktNum; - ic_statistics.recvPktNum += metrics.recvPktNum; - ic_statistics.disorderedPktNum += metrics.disorderedPktNum; - ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; - ic_statistics.recvAckNum += metrics.recvAckNum; - ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; + if (Gp_interconnect_type != INTERCONNECT_TYPE_UDPIFC) + { + return; + } + if (!proto_verified || gp_command_count == 0 || + !config.enable_collector() || config.filter_user(get_user_name())) + { + return; + } + // we also would like to know nesting level here and filter queries BUT we + // don't have this kind of information from this callback. Will have to + // collect stats anyways and throw it away later, if necessary + auto metrics = UDPIFCGetICStats(); + ic_statistics.totalRecvQueueSize += metrics.totalRecvQueueSize; + ic_statistics.recvQueueSizeCountingTime += + metrics.recvQueueSizeCountingTime; + ic_statistics.totalCapacity += metrics.totalCapacity; + ic_statistics.capacityCountingTime += metrics.capacityCountingTime; + ic_statistics.totalBuffers += metrics.totalBuffers; + ic_statistics.bufferCountingTime += metrics.bufferCountingTime; + ic_statistics.activeConnectionsNum += metrics.activeConnectionsNum; + ic_statistics.retransmits += metrics.retransmits; + ic_statistics.startupCachedPktNum += metrics.startupCachedPktNum; + ic_statistics.mismatchNum += metrics.mismatchNum; + ic_statistics.crcErrors += metrics.crcErrors; + ic_statistics.sndPktNum += metrics.sndPktNum; + ic_statistics.recvPktNum += metrics.recvPktNum; + ic_statistics.disorderedPktNum += metrics.disorderedPktNum; + ic_statistics.duplicatedPktNum += metrics.duplicatedPktNum; + ic_statistics.recvAckNum += metrics.recvAckNum; + ic_statistics.statusQueryMsgNum += metrics.statusQueryMsgNum; #endif } -void EventSender::analyze_stats_collect(QueryDesc *query_desc) { - if (!verify_query(query_desc, QueryState::END, false /* utility */)) { - return; - } - if (Gp_role != GP_ROLE_DISPATCH) { - return; - } - if (!query_desc->totaltime || !need_collect_analyze()) { - return; - } - // Make sure stats accumulation is done. - // (Note: it's okay if several levels of hook all do this.) - gpdb::instr_end_loop(query_desc->totaltime); - - double ms = query_desc->totaltime->total * 1000.0; - if (ms >= config.min_analyze_time()) { - auto &query = get_query(query_desc); - auto *query_msg = query.message.get(); - set_analyze_plan_text(query_desc, query_msg, config); - } +void +EventSender::analyze_stats_collect(QueryDesc *query_desc) +{ + if (!verify_query(query_desc, QueryState::END, false /* utility */)) + { + return; + } + if (Gp_role != GP_ROLE_DISPATCH) + { + return; + } + if (!query_desc->totaltime || !need_collect_analyze()) + { + return; + } + // Make sure stats accumulation is done. + // (Note: it's okay if several levels of hook all do this.) + gpdb::instr_end_loop(query_desc->totaltime); + + double ms = query_desc->totaltime->total * 1000.0; + if (ms >= config.min_analyze_time()) + { + auto &query = get_query(query_desc); + auto *query_msg = query.message.get(); + set_analyze_plan_text(query_desc, query_msg, config); + } } -EventSender::EventSender() { - // Perform initial sync to get default GUC values - config.sync(); - - try { - GOOGLE_PROTOBUF_VERIFY_VERSION; - proto_verified = true; - } catch (const std::exception &e) { - ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", e.what()))); - } +EventSender::EventSender() +{ + // Perform initial sync to get default GUC values + config.sync(); + + try + { + GOOGLE_PROTOBUF_VERIFY_VERSION; + proto_verified = true; + } + catch (const std::exception &e) + { + ereport(INFO, (errmsg("GPSC protobuf version mismatch is detected %s", + e.what()))); + } #ifdef IC_TEARDOWN_HOOK - memset(&ic_statistics, 0, sizeof(ICStatistics)); + memset(&ic_statistics, 0, sizeof(ICStatistics)); #endif } -EventSender::~EventSender() { - for (const auto &[qkey, _] : queries) { - ereport(LOG, (errmsg("GPSC query with missing done event: " - "tmid=%d ssid=%d ccnt=%d nlvl=%d", - qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); - } +EventSender::~EventSender() +{ + for (const auto &[qkey, _] : queries) + { + ereport(LOG, + (errmsg("GPSC query with missing done event: " + "tmid=%d ssid=%d ccnt=%d nlvl=%d", + qkey.tmid, qkey.ssid, qkey.ccnt, qkey.nesting_level))); + } } // That's basically a very simplistic state machine to fix or highlight any bugs // coming from GP -void EventSender::update_query_state(QueryItem &query, QueryState new_state, - bool utility, bool success) { - switch (new_state) { - case QueryState::SUBMIT: - Assert(false); - break; - case QueryState::START: - if (query.state == QueryState::SUBMIT) { - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_START); - } else { - Assert(false); - } - break; - case QueryState::END: - // Example of below assert triggering: CURSOR closes before ever being - // executed Assert(query->state == QueryState::START || - // IsAbortInProgress()); - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_END); - break; - case QueryState::DONE: - Assert(query.state == QueryState::END || !success || utility); - query.message->set_query_status(gpsc::QueryStatus::QUERY_STATUS_DONE); - break; - default: - Assert(false); - } - query.state = new_state; +void +EventSender::update_query_state(QueryItem &query, QueryState new_state, + bool utility, bool success) +{ + switch (new_state) + { + case QueryState::SUBMIT: + Assert(false); + break; + case QueryState::START: + if (query.state == QueryState::SUBMIT) + { + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_START); + } + else + { + Assert(false); + } + break; + case QueryState::END: + // Example of below assert triggering: CURSOR closes before ever being + // executed Assert(query->state == QueryState::START || + // IsAbortInProgress()); + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_END); + break; + case QueryState::DONE: + Assert(query.state == QueryState::END || !success || utility); + query.message->set_query_status( + gpsc::QueryStatus::QUERY_STATUS_DONE); + break; + default: + Assert(false); + } + query.state = new_state; } -EventSender::QueryItem &EventSender::get_query(QueryDesc *query_desc) { - if (!qdesc_submitted(query_desc)) { - ereport(WARNING, - (errmsg("GPSC attempting to get query that was not submitted"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - throw std::runtime_error("Attempting to get query that was not submitted"); - } - return queries.find(QueryKey::from_qdesc(query_desc))->second; +EventSender::QueryItem & +EventSender::get_query(QueryDesc *query_desc) +{ + if (!qdesc_submitted(query_desc)) + { + ereport( + WARNING, + (errmsg("GPSC attempting to get query that was not submitted"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + throw std::runtime_error( + "Attempting to get query that was not submitted"); + } + return queries.find(QueryKey::from_qdesc(query_desc))->second; } -void EventSender::submit_query(QueryDesc *query_desc) { - if (query_desc->gpsc_query_key) { - ereport(WARNING, - (errmsg("GPSC trying to submit already submitted query"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } - QueryKey::register_qkey(query_desc, nesting_level); - auto key = QueryKey::from_qdesc(query_desc); - auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); - if (!inserted) { - ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); - ereport(DEBUG3, - (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); - } +void +EventSender::submit_query(QueryDesc *query_desc) +{ + if (query_desc->gpsc_query_key) + { + ereport(WARNING, + (errmsg("GPSC trying to submit already submitted query"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + } + QueryKey::register_qkey(query_desc, nesting_level); + auto key = QueryKey::from_qdesc(query_desc); + auto [_, inserted] = queries.emplace(key, QueryItem(QueryState::SUBMIT)); + if (!inserted) + { + ereport(WARNING, (errmsg("GPSC duplicate query submit detected"))); + ereport(DEBUG3, + (errmsg("GPSC query sourceText: %s", query_desc->sourceText))); + } } -void EventSender::update_nested_counters(QueryDesc *query_desc) { - if (!is_top_level_query(query_desc, nesting_level)) { - auto &query = get_query(query_desc); - nested_calls++; - double end_time = protots_to_double(query.message->end_time()); - double start_time = protots_to_double(query.message->start_time()); - if (end_time >= start_time) { - nested_timing += end_time - start_time; - } else { - ereport(WARNING, (errmsg("GPSC query start_time > end_time (%f > %f)", - start_time, end_time))); - ereport(DEBUG3, - (errmsg("GPSC nested query text %s", query_desc->sourceText))); - } - } +void +EventSender::update_nested_counters(QueryDesc *query_desc) +{ + if (!is_top_level_query(query_desc, nesting_level)) + { + auto &query = get_query(query_desc); + nested_calls++; + double end_time = protots_to_double(query.message->end_time()); + double start_time = protots_to_double(query.message->start_time()); + if (end_time >= start_time) + { + nested_timing += end_time - start_time; + } + else + { + ereport(WARNING, + (errmsg("GPSC query start_time > end_time (%f > %f)", + start_time, end_time))); + ereport(DEBUG3, (errmsg("GPSC nested query text %s", + query_desc->sourceText))); + } + } } -bool EventSender::qdesc_submitted(QueryDesc *query_desc) { - if (query_desc->gpsc_query_key == NULL) { - return false; - } - return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); +bool +EventSender::qdesc_submitted(QueryDesc *query_desc) +{ + if (query_desc->gpsc_query_key == NULL) + { + return false; + } + return queries.find(QueryKey::from_qdesc(query_desc)) != queries.end(); } -bool EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) { - return need_report_nested_query() || - is_top_level_query(query_desc, nesting_level); +bool +EventSender::nesting_is_valid(QueryDesc *query_desc, int nesting_level) +{ + return need_report_nested_query() || + is_top_level_query(query_desc, nesting_level); } -bool EventSender::need_report_nested_query() { - return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; +bool +EventSender::need_report_nested_query() +{ + return config.report_nested_queries() && Gp_role == GP_ROLE_DISPATCH; } -bool EventSender::filter_query(QueryDesc *query_desc) { - return gp_command_count == 0 || query_desc->sourceText == nullptr || - !config.enable_collector() || config.filter_user(get_user_name()); +bool +EventSender::filter_query(QueryDesc *query_desc) +{ + return gp_command_count == 0 || query_desc->sourceText == nullptr || + !config.enable_collector() || config.filter_user(get_user_name()); } EventSender::QueryItem::QueryItem(QueryState st) - : message(std::make_unique()), state(st) {} + : message(std::make_unique()), state(st) +{ +} diff --git a/gpcontrib/gp_stats_collector/src/EventSender.h b/gpcontrib/gp_stats_collector/src/EventSender.h index 154c2c0dceb..2651a020593 100644 --- a/gpcontrib/gp_stats_collector/src/EventSender.h +++ b/gpcontrib/gp_stats_collector/src/EventSender.h @@ -25,11 +25,12 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef EVENTSENDER_H +#define EVENTSENDER_H #include -#include #include +#include #define typeid __typeid extern "C" { @@ -40,12 +41,13 @@ extern "C" { } #undef typeid -#include "memory/gpdbwrappers.h" #include "Config.h" +#include "memory/gpdbwrappers.h" class UDSConnector; struct QueryDesc; -namespace gpsc { +namespace gpsc +{ class SetQueryReq; } @@ -53,116 +55,149 @@ class SetQueryReq; extern void gp_gettmid(int32 *); -struct QueryKey { - int tmid; - int ssid; - int ccnt; - int nesting_level; - uintptr_t query_desc_addr; - - bool operator==(const QueryKey &other) const { - return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == - std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, - other.query_desc_addr); - } - - static void register_qkey(QueryDesc *query_desc, size_t nesting_level) { - query_desc->gpsc_query_key = - (GpscQueryKey *)gpdb::palloc0(sizeof(GpscQueryKey)); - int32 tmid; - gp_gettmid(&tmid); - query_desc->gpsc_query_key->tmid = tmid; - query_desc->gpsc_query_key->ssid = gp_session_id; - query_desc->gpsc_query_key->ccnt = gp_command_count; - query_desc->gpsc_query_key->nesting_level = nesting_level; - query_desc->gpsc_query_key->query_desc_addr = (uintptr_t)query_desc; - } - - static QueryKey from_qdesc(QueryDesc *query_desc) { - return { - .tmid = query_desc->gpsc_query_key->tmid, - .ssid = query_desc->gpsc_query_key->ssid, - .ccnt = query_desc->gpsc_query_key->ccnt, - .nesting_level = query_desc->gpsc_query_key->nesting_level, - .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, - }; - } +struct QueryKey +{ + int tmid; + int ssid; + int ccnt; + int nesting_level; + uintptr_t query_desc_addr; + + bool + operator==(const QueryKey &other) const + { + return std::tie(tmid, ssid, ccnt, nesting_level, query_desc_addr) == + std::tie(other.tmid, other.ssid, other.ccnt, other.nesting_level, + other.query_desc_addr); + } + + static void + register_qkey(QueryDesc *query_desc, size_t nesting_level) + { + query_desc->gpsc_query_key = + (GpscQueryKey *) gpdb::palloc0(sizeof(GpscQueryKey)); + int32 tmid; + gp_gettmid(&tmid); + query_desc->gpsc_query_key->tmid = tmid; + query_desc->gpsc_query_key->ssid = gp_session_id; + query_desc->gpsc_query_key->ccnt = gp_command_count; + query_desc->gpsc_query_key->nesting_level = nesting_level; + query_desc->gpsc_query_key->query_desc_addr = (uintptr_t) query_desc; + } + + static QueryKey + from_qdesc(QueryDesc *query_desc) + { + return { + .tmid = query_desc->gpsc_query_key->tmid, + .ssid = query_desc->gpsc_query_key->ssid, + .ccnt = query_desc->gpsc_query_key->ccnt, + .nesting_level = query_desc->gpsc_query_key->nesting_level, + .query_desc_addr = query_desc->gpsc_query_key->query_desc_addr, + }; + } }; // https://www.boost.org/doc/libs/1_35_0/doc/html/boost/hash_combine_id241013.html -template inline void hash_combine(std::size_t &seed, const T &v) { - std::hash hasher; - seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +template +inline void +hash_combine(std::size_t &seed, const T &v) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); } -namespace std { -template <> struct hash { - size_t operator()(const QueryKey &k) const noexcept { - size_t seed = hash{}(k.tmid); - hash_combine(seed, k.ssid); - hash_combine(seed, k.ccnt); - hash_combine(seed, k.nesting_level); - uintptr_t addr = k.query_desc_addr; - if constexpr (SIZE_MAX < UINTPTR_MAX) { - addr %= SIZE_MAX; - } - hash_combine(seed, addr); - return seed; - } +namespace std +{ +template <> +struct hash +{ + size_t + operator()(const QueryKey &k) const noexcept + { + size_t seed = hash{}(k.tmid); + hash_combine(seed, k.ssid); + hash_combine(seed, k.ccnt); + hash_combine(seed, k.nesting_level); + uintptr_t addr = k.query_desc_addr; + if constexpr (SIZE_MAX < UINTPTR_MAX) + { + addr %= SIZE_MAX; + } + hash_combine(seed, addr); + return seed; + } }; -} // namespace std +} // namespace std -class EventSender { +class EventSender +{ public: - void executor_before_start(QueryDesc *query_desc, int eflags); - void executor_after_start(QueryDesc *query_desc, int eflags); - void executor_end(QueryDesc *query_desc); - void query_metrics_collect(QueryMetricsStatus status, void *arg, bool utility, - ErrorData *edata = NULL); - void ic_metrics_collect(); - void analyze_stats_collect(QueryDesc *query_desc); - void incr_depth() { nesting_level++; } - void decr_depth() { nesting_level--; } - EventSender(); - ~EventSender(); + void executor_before_start(QueryDesc *query_desc, int eflags); + void executor_after_start(QueryDesc *query_desc, int eflags); + void executor_end(QueryDesc *query_desc); + void query_metrics_collect(QueryMetricsStatus status, void *arg, + bool utility, ErrorData *edata = NULL); + void ic_metrics_collect(); + void analyze_stats_collect(QueryDesc *query_desc); + void + incr_depth() + { + nesting_level++; + } + void + decr_depth() + { + nesting_level--; + } + EventSender(); + ~EventSender(); private: - enum QueryState { SUBMIT, START, END, DONE }; - - struct QueryItem { - std::unique_ptr message; - QueryState state; - - explicit QueryItem(QueryState st); - }; - - bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, - bool utility); - bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); - void update_query_state(QueryItem &query, QueryState new_state, bool utility, - bool success = true); - QueryItem &get_query(QueryDesc *query_desc); - void submit_query(QueryDesc *query_desc); - void collect_query_submit(QueryDesc *query_desc, bool utility); - void report_query_done(QueryDesc *query_desc, QueryItem &query, - QueryMetricsStatus status, bool utility, - ErrorData *edata = NULL); - void collect_query_done(QueryDesc *query_desc, bool utility, - QueryMetricsStatus status, ErrorData *edata = NULL); - void update_nested_counters(QueryDesc *query_desc); - bool qdesc_submitted(QueryDesc *query_desc); - bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); - bool need_report_nested_query(); - bool filter_query(QueryDesc *query_desc); - - bool proto_verified = false; - int nesting_level = 0; - int64_t nested_calls = 0; - double nested_timing = 0; + enum QueryState + { + SUBMIT, + START, + END, + DONE + }; + + struct QueryItem + { + std::unique_ptr message; + QueryState state; + + explicit QueryItem(QueryState st); + }; + + bool log_query_req(const gpsc::SetQueryReq &req, const std::string &event, + bool utility); + bool verify_query(QueryDesc *query_desc, QueryState state, bool utility); + void update_query_state(QueryItem &query, QueryState new_state, + bool utility, bool success = true); + QueryItem &get_query(QueryDesc *query_desc); + void submit_query(QueryDesc *query_desc); + void collect_query_submit(QueryDesc *query_desc, bool utility); + void report_query_done(QueryDesc *query_desc, QueryItem &query, + QueryMetricsStatus status, bool utility, + ErrorData *edata = NULL); + void collect_query_done(QueryDesc *query_desc, bool utility, + QueryMetricsStatus status, ErrorData *edata = NULL); + void update_nested_counters(QueryDesc *query_desc); + bool qdesc_submitted(QueryDesc *query_desc); + bool nesting_is_valid(QueryDesc *query_desc, int nesting_level); + bool need_report_nested_query(); + bool filter_query(QueryDesc *query_desc); + + bool proto_verified = false; + int nesting_level = 0; + int64_t nested_calls = 0; + double nested_timing = 0; #ifdef IC_TEARDOWN_HOOK - ICStatistics ic_statistics; + ICStatistics ic_statistics; #endif - std::unordered_map queries; + std::unordered_map queries; - Config config; -}; \ No newline at end of file + Config config; +}; +#endif /* EVENTSENDER_H */ diff --git a/gpcontrib/gp_stats_collector/src/GpscStat.cpp b/gpcontrib/gp_stats_collector/src/GpscStat.cpp index c4029f085cf..151cfd87c02 100644 --- a/gpcontrib/gp_stats_collector/src/GpscStat.cpp +++ b/gpcontrib/gp_stats_collector/src/GpscStat.cpp @@ -38,81 +38,117 @@ extern "C" { #include "storage/spin.h" } -namespace { -struct ProtectedData { - slock_t mutex; - GpscStat::Data data; +namespace +{ +struct ProtectedData +{ + slock_t mutex; + GpscStat::Data data; }; shmem_startup_hook_type prev_shmem_startup_hook = NULL; ProtectedData *data = nullptr; -void gpsc_shmem_startup() { - if (prev_shmem_startup_hook) - prev_shmem_startup_hook(); - LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); - bool found; - data = reinterpret_cast( - ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); - if (!found) { - SpinLockInit(&data->mutex); - data->data = GpscStat::Data(); - } - LWLockRelease(AddinShmemInitLock); +void +gpsc_shmem_startup() +{ + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + bool found; + data = reinterpret_cast( + ShmemInitStruct("gpsc_stat_messages", sizeof(ProtectedData), &found)); + if (!found) + { + SpinLockInit(&data->mutex); + data->data = GpscStat::Data(); + } + LWLockRelease(AddinShmemInitLock); } -class LockGuard { +class LockGuard +{ public: - LockGuard(slock_t *mutex) : mutex_(mutex) { SpinLockAcquire(mutex_); } - ~LockGuard() { SpinLockRelease(mutex_); } + LockGuard(slock_t *mutex) : mutex_(mutex) + { + SpinLockAcquire(mutex_); + } + ~LockGuard() + { + SpinLockRelease(mutex_); + } private: - slock_t *mutex_; + slock_t *mutex_; }; -} // namespace - -void GpscStat::init() { - if (!process_shared_preload_libraries_in_progress) - return; - RequestAddinShmemSpace(sizeof(ProtectedData)); - prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = gpsc_shmem_startup; +} // namespace + +void +GpscStat::init() +{ + if (!process_shared_preload_libraries_in_progress) + return; + RequestAddinShmemSpace(sizeof(ProtectedData)); + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = gpsc_shmem_startup; } -void GpscStat::deinit() { shmem_startup_hook = prev_shmem_startup_hook; } +void +GpscStat::deinit() +{ + shmem_startup_hook = prev_shmem_startup_hook; +} -void GpscStat::reset() { - LockGuard lg(&data->mutex); - data->data = GpscStat::Data(); +void +GpscStat::reset() +{ + LockGuard lg(&data->mutex); + data->data = GpscStat::Data(); } -void GpscStat::report_send(int32_t msg_size) { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +void +GpscStat::report_send(int32_t msg_size) +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.max_message_size = + std::max(msg_size, data->data.max_message_size); } -void GpscStat::report_bad_connection() { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_connects++; +void +GpscStat::report_bad_connection() +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_connects++; } -void GpscStat::report_bad_send(int32_t msg_size) { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_sends++; - data->data.max_message_size = std::max(msg_size, data->data.max_message_size); +void +GpscStat::report_bad_send(int32_t msg_size) +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_sends++; + data->data.max_message_size = + std::max(msg_size, data->data.max_message_size); } -void GpscStat::report_error() { - LockGuard lg(&data->mutex); - data->data.total++; - data->data.failed_other++; +void +GpscStat::report_error() +{ + LockGuard lg(&data->mutex); + data->data.total++; + data->data.failed_other++; } -GpscStat::Data GpscStat::get_stats() { - LockGuard lg(&data->mutex); - return data->data; +GpscStat::Data +GpscStat::get_stats() +{ + LockGuard lg(&data->mutex); + return data->data; } -bool GpscStat::loaded() { return data != nullptr; } +bool +GpscStat::loaded() +{ + return data != nullptr; +} diff --git a/gpcontrib/gp_stats_collector/src/GpscStat.h b/gpcontrib/gp_stats_collector/src/GpscStat.h index af1a1261776..d82930c7b5b 100644 --- a/gpcontrib/gp_stats_collector/src/GpscStat.h +++ b/gpcontrib/gp_stats_collector/src/GpscStat.h @@ -25,24 +25,28 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef GPSCSTAT_H +#define GPSCSTAT_H #include -class GpscStat { +class GpscStat +{ public: - struct Data { - int64_t total, failed_sends, failed_connects, failed_other; - int32_t max_message_size; - }; + struct Data + { + int64_t total, failed_sends, failed_connects, failed_other; + int32_t max_message_size; + }; - static void init(); - static void deinit(); - static void reset(); - static void report_send(int32_t msg_size); - static void report_bad_connection(); - static void report_bad_send(int32_t msg_size); - static void report_error(); - static Data get_stats(); - static bool loaded(); -}; \ No newline at end of file + static void init(); + static void deinit(); + static void reset(); + static void report_send(int32_t msg_size); + static void report_bad_connection(); + static void report_bad_send(int32_t msg_size); + static void report_error(); + static Data get_stats(); + static bool loaded(); +}; +#endif /* GPSCSTAT_H */ diff --git a/gpcontrib/gp_stats_collector/src/PgUtils.cpp b/gpcontrib/gp_stats_collector/src/PgUtils.cpp index 3dbee97061b..c473cc383f2 100644 --- a/gpcontrib/gp_stats_collector/src/PgUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/PgUtils.cpp @@ -30,39 +30,46 @@ #include "memory/gpdbwrappers.h" extern "C" { -#include "commands/resgroupcmds.h" #include "cdb/cdbvars.h" +#include "commands/resgroupcmds.h" } -std::string get_user_name() { - // username is allocated on stack, we don't need to pfree it. - const char *username = - gpdb::get_config_option("session_authorization", false, false); - return username ? std::string(username) : ""; +std::string +get_user_name() +{ + // username is allocated on stack, we don't need to pfree it. + const char *username = + gpdb::get_config_option("session_authorization", false, false); + return username ? std::string(username) : ""; } -std::string get_db_name() { - char *dbname = gpdb::get_database_name(MyDatabaseId); - if (dbname) { - std::string result(dbname); - gpdb::pfree(dbname); - return result; - } - return ""; +std::string +get_db_name() +{ + char *dbname = gpdb::get_database_name(MyDatabaseId); + if (dbname) + { + std::string result(dbname); + gpdb::pfree(dbname); + return result; + } + return ""; } -std::string get_rg_name() { - auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); - if (!OidIsValid(groupId)) - return ""; +std::string +get_rg_name() +{ + auto groupId = gpdb::get_rg_id_by_session_id(MySessionState->sessionId); + if (!OidIsValid(groupId)) + return ""; - char *rgname = gpdb::get_rg_name_for_id(groupId); - if (rgname == nullptr) - return ""; + char *rgname = gpdb::get_rg_name_for_id(groupId); + if (rgname == nullptr) + return ""; - std::string result(rgname); - gpdb::pfree(rgname); - return result; + std::string result(rgname); + gpdb::pfree(rgname); + return result; } /** @@ -86,9 +93,12 @@ std::string get_rg_name() { * segment sees those as top-level. */ -bool is_top_level_query(QueryDesc *query_desc, int nesting_level) { - if (query_desc->gpsc_query_key == NULL) { - return nesting_level == 0; - } - return query_desc->gpsc_query_key->nesting_level == 0; +bool +is_top_level_query(QueryDesc *query_desc, int nesting_level) +{ + if (query_desc->gpsc_query_key == NULL) + { + return nesting_level == 0; + } + return query_desc->gpsc_query_key->nesting_level == 0; } diff --git a/gpcontrib/gp_stats_collector/src/ProcStats.cpp b/gpcontrib/gp_stats_collector/src/ProcStats.cpp index 9c557879fc6..e308b30dfa5 100644 --- a/gpcontrib/gp_stats_collector/src/ProcStats.cpp +++ b/gpcontrib/gp_stats_collector/src/ProcStats.cpp @@ -26,100 +26,119 @@ */ #include "ProcStats.h" -#include "gpsc_metrics.pb.h" -#include #include +#include #include +#include "gpsc_metrics.pb.h" extern "C" { #include "postgres.h" #include "utils/elog.h" } -namespace { -#define FILL_IO_STAT(stat_name) \ - uint64_t stat_name; \ - proc_stat >> tmp >> stat_name; \ - stats->set_##stat_name(stat_name - stats->stat_name()); +namespace +{ +#define FILL_IO_STAT(stat_name) \ + uint64_t stat_name; \ + proc_stat >> tmp >> stat_name; \ + stats->set_##stat_name(stat_name - stats->stat_name()); -void fill_io_stats(gpsc::SystemStat *stats) { - std::ifstream proc_stat("/proc/self/io"); - std::string tmp; - FILL_IO_STAT(rchar); - FILL_IO_STAT(wchar); - FILL_IO_STAT(syscr); - FILL_IO_STAT(syscw); - FILL_IO_STAT(read_bytes); - FILL_IO_STAT(write_bytes); - FILL_IO_STAT(cancelled_write_bytes); +void +fill_io_stats(gpsc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/io"); + std::string tmp; + FILL_IO_STAT(rchar); + FILL_IO_STAT(wchar); + FILL_IO_STAT(syscr); + FILL_IO_STAT(syscw); + FILL_IO_STAT(read_bytes); + FILL_IO_STAT(write_bytes); + FILL_IO_STAT(cancelled_write_bytes); } -void fill_cpu_stats(gpsc::SystemStat *stats) { - static const int UTIME_ID = 13; - static const int STIME_ID = 14; - static const int VSIZE_ID = 22; - static const int RSS_ID = 23; - static const double tps = sysconf(_SC_CLK_TCK); +void +fill_cpu_stats(gpsc::SystemStat *stats) +{ + static const int UTIME_ID = 13; + static const int STIME_ID = 14; + static const int VSIZE_ID = 22; + static const int RSS_ID = 23; + static const double tps = sysconf(_SC_CLK_TCK); - std::ifstream proc_stat("/proc/self/stat"); - std::string trash; - for (int i = 0; i <= RSS_ID; ++i) { - switch (i) { - case UTIME_ID: - double utime; - proc_stat >> utime; - stats->set_usertimeseconds(utime / tps - stats->usertimeseconds()); - break; - case STIME_ID: - double stime; - proc_stat >> stime; - stats->set_kerneltimeseconds(stime / tps - stats->kerneltimeseconds()); - break; - case VSIZE_ID: - uint64_t vsize; - proc_stat >> vsize; - stats->set_vsize(vsize); - break; - case RSS_ID: - uint64_t rss; - proc_stat >> rss; - // NOTE: this is a double AFAIU, need to double-check - stats->set_rss(rss); - break; - default: - proc_stat >> trash; - } - } + std::ifstream proc_stat("/proc/self/stat"); + std::string trash; + for (int i = 0; i <= RSS_ID; ++i) + { + switch (i) + { + case UTIME_ID: + double utime; + proc_stat >> utime; + stats->set_usertimeseconds(utime / tps - + stats->usertimeseconds()); + break; + case STIME_ID: + double stime; + proc_stat >> stime; + stats->set_kerneltimeseconds(stime / tps - + stats->kerneltimeseconds()); + break; + case VSIZE_ID: + uint64_t vsize; + proc_stat >> vsize; + stats->set_vsize(vsize); + break; + case RSS_ID: + uint64_t rss; + proc_stat >> rss; + // NOTE: this is a double AFAIU, need to double-check + stats->set_rss(rss); + break; + default: + proc_stat >> trash; + } + } } -void fill_status_stats(gpsc::SystemStat *stats) { - std::ifstream proc_stat("/proc/self/status"); - std::string key, measure; - while (proc_stat >> key) { - if (key == "VmPeak:") { - uint64_t value; - proc_stat >> value; - stats->set_vmpeakkb(value); - proc_stat >> measure; - if (measure != "kB") { - throw std::runtime_error("Expected memory sizes in kB, but got in " + - measure); - } - } else if (key == "VmSize:") { - uint64_t value; - proc_stat >> value; - stats->set_vmsizekb(value); - if (measure != "kB") { - throw std::runtime_error("Expected memory sizes in kB, but got in " + - measure); - } - } - } +void +fill_status_stats(gpsc::SystemStat *stats) +{ + std::ifstream proc_stat("/proc/self/status"); + std::string key, measure; + while (proc_stat >> key) + { + if (key == "VmPeak:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmpeakkb(value); + proc_stat >> measure; + if (measure != "kB") + { + throw std::runtime_error( + "Expected memory sizes in kB, but got in " + measure); + } + } + else if (key == "VmSize:") + { + uint64_t value; + proc_stat >> value; + stats->set_vmsizekb(value); + if (measure != "kB") + { + throw std::runtime_error( + "Expected memory sizes in kB, but got in " + measure); + } + } + } } -} // namespace +} // namespace -void fill_self_stats(gpsc::SystemStat *stats) { - fill_io_stats(stats); - fill_cpu_stats(stats); - fill_status_stats(stats); +void +fill_self_stats(gpsc::SystemStat *stats) +{ + fill_io_stats(stats); + fill_cpu_stats(stats); + fill_status_stats(stats); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/ProcStats.h b/gpcontrib/gp_stats_collector/src/ProcStats.h index 4473125f875..8b83dbfef02 100644 --- a/gpcontrib/gp_stats_collector/src/ProcStats.h +++ b/gpcontrib/gp_stats_collector/src/ProcStats.h @@ -25,10 +25,13 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef PROCSTATS_H +#define PROCSTATS_H -namespace gpsc { +namespace gpsc +{ class SystemStat; } -void fill_self_stats(gpsc::SystemStat *stats); \ No newline at end of file +void fill_self_stats(gpsc::SystemStat *stats); +#endif /* PROCSTATS_H */ diff --git a/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp index c9ceff4739b..b22f580303e 100644 --- a/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.cpp @@ -26,9 +26,9 @@ */ #include "ProtoUtils.h" +#include "Config.h" #include "PgUtils.h" #include "ProcStats.h" -#include "Config.h" #include "memory/gpdbwrappers.h" #define typeid __typeid @@ -53,265 +53,323 @@ extern "C" { extern void gp_gettmid(int32 *); -namespace { +namespace +{ constexpr uint8_t UTF8_CONTINUATION_BYTE_MASK = (1 << 7) | (1 << 6); constexpr uint8_t UTF8_CONTINUATION_BYTE = (1 << 7); constexpr uint8_t UTF8_MAX_SYMBOL_BYTES = 4; // Returns true if byte is the starting byte of utf8 // character, false if byte is the continuation (10xxxxxx). -inline bool utf8_start_byte(uint8_t byte) { - return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; +inline bool +utf8_start_byte(uint8_t byte) +{ + return (byte & UTF8_CONTINUATION_BYTE_MASK) != UTF8_CONTINUATION_BYTE; } -} // namespace +} // namespace -google::protobuf::Timestamp current_ts() { - google::protobuf::Timestamp current_ts; - struct timeval tv; - gettimeofday(&tv, nullptr); - current_ts.set_seconds(tv.tv_sec); - current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); - return current_ts; +google::protobuf::Timestamp +current_ts() +{ + google::protobuf::Timestamp current_ts; + struct timeval tv; + gettimeofday(&tv, nullptr); + current_ts.set_seconds(tv.tv_sec); + current_ts.set_nanos(static_cast(tv.tv_usec * 1000)); + return current_ts; } -void set_query_key(gpsc::QueryKey *key) { - key->set_ccnt(gp_command_count); - key->set_ssid(gp_session_id); - int32 tmid = 0; - gp_gettmid(&tmid); - key->set_tmid(tmid); +void +set_query_key(gpsc::QueryKey *key) +{ + key->set_ccnt(gp_command_count); + key->set_ssid(gp_session_id); + int32 tmid = 0; + gp_gettmid(&tmid); + key->set_tmid(tmid); } -void set_segment_key(gpsc::SegmentKey *key) { - key->set_dbid(GpIdentity.dbid); - key->set_segindex(GpIdentity.segindex); +void +set_segment_key(gpsc::SegmentKey *key) +{ + key->set_dbid(GpIdentity.dbid); + key->set_segindex(GpIdentity.segindex); } -std::string trim_str_shrink_utf8(const char *str, size_t len, size_t lim) { - if (unlikely(str == nullptr)) { - return std::string(); - } - if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) { - return std::string(str, std::min(len, lim)); - } +std::string +trim_str_shrink_utf8(const char *str, size_t len, size_t lim) +{ + if (unlikely(str == nullptr)) + { + return std::string(); + } + if (likely(len <= lim || GetDatabaseEncoding() != PG_UTF8)) + { + return std::string(str, std::min(len, lim)); + } - // Handle trimming of utf8 correctly, do not cut multi-byte characters. - size_t cut_pos = lim; - size_t visited_bytes = 1; - while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) { - if (utf8_start_byte(static_cast(str[cut_pos]))) { - break; - } - ++visited_bytes; - --cut_pos; - } + // Handle trimming of utf8 correctly, do not cut multi-byte characters. + size_t cut_pos = lim; + size_t visited_bytes = 1; + while (visited_bytes < UTF8_MAX_SYMBOL_BYTES && cut_pos > 0) + { + if (utf8_start_byte(static_cast(str[cut_pos]))) + { + break; + } + ++visited_bytes; + --cut_pos; + } - return std::string(str, cut_pos); + return std::string(str, cut_pos); } -void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config) { - if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) { - auto qi = req->mutable_query_info(); - qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER - ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER - : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_explain_state(query_desc, true); - if (es.str) { - *qi->mutable_plan_text() = trim_str_shrink_utf8(es.str->data, es.str->len, - config.max_plan_size()); - StringInfo norm_plan = gpdb::gen_normplan(es.str->data); - if (norm_plan) { - *qi->mutable_template_plan_text() = trim_str_shrink_utf8( - norm_plan->data, norm_plan->len, config.max_plan_size()); - qi->set_plan_id( - hash_any((unsigned char *)norm_plan->data, norm_plan->len)); - gpdb::pfree(norm_plan->data); - } - qi->set_query_id(query_desc->plannedstmt->queryId); - gpdb::pfree(es.str->data); - } - gpdb::mem_ctx_switch_to(oldcxt); - } +void +set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) +{ + if (Gp_role == GP_ROLE_DISPATCH && query_desc->plannedstmt) + { + auto qi = req->mutable_query_info(); + qi->set_generator(query_desc->plannedstmt->planGen == PLANGEN_OPTIMIZER + ? gpsc::PlanGenerator::PLAN_GENERATOR_OPTIMIZER + : gpsc::PlanGenerator::PLAN_GENERATOR_PLANNER); + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_explain_state(query_desc, true); + if (es.str) + { + *qi->mutable_plan_text() = trim_str_shrink_utf8( + es.str->data, es.str->len, config.max_plan_size()); + StringInfo norm_plan = gpdb::gen_normplan(es.str->data); + if (norm_plan) + { + *qi->mutable_template_plan_text() = trim_str_shrink_utf8( + norm_plan->data, norm_plan->len, config.max_plan_size()); + qi->set_plan_id(hash_any((unsigned char *) norm_plan->data, + norm_plan->len)); + gpdb::pfree(norm_plan->data); + } + qi->set_query_id(query_desc->plannedstmt->queryId); + gpdb::pfree(es.str->data); + } + gpdb::mem_ctx_switch_to(oldcxt); + } } -void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config) { - if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) { - auto qi = req->mutable_query_info(); - *qi->mutable_query_text() = trim_str_shrink_utf8( - query_desc->sourceText, strlen(query_desc->sourceText), - config.max_text_size()); - char *norm_query = gpdb::gen_normquery(query_desc->sourceText); - if (norm_query) { - *qi->mutable_template_query_text() = trim_str_shrink_utf8( - norm_query, strlen(norm_query), config.max_text_size()); - gpdb::pfree(norm_query); - } - } +void +set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, + const Config &config) +{ + if (Gp_role == GP_ROLE_DISPATCH && query_desc->sourceText) + { + auto qi = req->mutable_query_info(); + *qi->mutable_query_text() = trim_str_shrink_utf8( + query_desc->sourceText, strlen(query_desc->sourceText), + config.max_text_size()); + char *norm_query = gpdb::gen_normquery(query_desc->sourceText); + if (norm_query) + { + *qi->mutable_template_query_text() = trim_str_shrink_utf8( + norm_query, strlen(norm_query), config.max_text_size()); + gpdb::pfree(norm_query); + } + } } -void clear_big_fields(gpsc::SetQueryReq *req) { - if (Gp_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->clear_plan_text(); - qi->clear_template_plan_text(); - qi->clear_query_text(); - qi->clear_template_query_text(); - qi->clear_analyze_text(); - } +void +clear_big_fields(gpsc::SetQueryReq *req) +{ + if (Gp_role == GP_ROLE_DISPATCH) + { + auto qi = req->mutable_query_info(); + qi->clear_plan_text(); + qi->clear_template_plan_text(); + qi->clear_query_text(); + qi->clear_template_query_text(); + qi->clear_analyze_text(); + } } -void set_query_info(gpsc::SetQueryReq *req) { - if (Gp_role == GP_ROLE_DISPATCH) { - auto qi = req->mutable_query_info(); - qi->set_username(get_user_name()); - if (IsTransactionState()) - qi->set_databasename(get_db_name()); - qi->set_rsgname(get_rg_name()); - } +void +set_query_info(gpsc::SetQueryReq *req) +{ + if (Gp_role == GP_ROLE_DISPATCH) + { + auto qi = req->mutable_query_info(); + qi->set_username(get_user_name()); + if (IsTransactionState()) + qi->set_databasename(get_db_name()); + qi->set_rsgname(get_rg_name()); + } } -void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) { - auto aqi = req->mutable_add_info(); - aqi->set_nested_level(nesting_level); +void +set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level) +{ + auto aqi = req->mutable_add_info(); + aqi->set_nested_level(nesting_level); } -void set_qi_slice_id(gpsc::SetQueryReq *req) { - auto aqi = req->mutable_add_info(); - aqi->set_slice_id(currentSliceId); +void +set_qi_slice_id(gpsc::SetQueryReq *req) +{ + auto aqi = req->mutable_add_info(); + aqi->set_slice_id(currentSliceId); } -void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, - const Config &config) { - auto aqi = req->mutable_add_info(); - *aqi->mutable_error_message() = - trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); +void +set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, + const Config &config) +{ + auto aqi = req->mutable_add_info(); + *aqi->mutable_error_message() = + trim_str_shrink_utf8(err_msg, strlen(err_msg), config.max_text_size()); } -void set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, - QueryDesc *query_desc, int nested_calls, - double nested_time) { - auto instrument = query_desc->planstate->instrument; - if (instrument) { - metrics->set_ntuples(instrument->ntuples); - metrics->set_nloops(instrument->nloops); - metrics->set_tuplecount(instrument->tuplecount); - metrics->set_firsttuple(instrument->firsttuple); - metrics->set_startup(instrument->startup); - metrics->set_total(instrument->total); - auto &buffusage = instrument->bufusage; - metrics->set_shared_blks_hit(buffusage.shared_blks_hit); - metrics->set_shared_blks_read(buffusage.shared_blks_read); - metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); - metrics->set_shared_blks_written(buffusage.shared_blks_written); - metrics->set_local_blks_hit(buffusage.local_blks_hit); - metrics->set_local_blks_read(buffusage.local_blks_read); - metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); - metrics->set_local_blks_written(buffusage.local_blks_written); - metrics->set_temp_blks_read(buffusage.temp_blks_read); - metrics->set_temp_blks_written(buffusage.temp_blks_written); - metrics->set_blk_read_time(INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); - metrics->set_blk_write_time( - INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); - } - if (query_desc->estate && query_desc->estate->motionlayer_context) { - MotionLayerState *mlstate = - (MotionLayerState *)query_desc->estate->motionlayer_context; - metrics->mutable_sent()->set_total_bytes(mlstate->stat_total_bytes_sent); - metrics->mutable_sent()->set_tuple_bytes(mlstate->stat_tuple_bytes_sent); - metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); - metrics->mutable_received()->set_total_bytes( - mlstate->stat_total_bytes_recvd); - metrics->mutable_received()->set_tuple_bytes( - mlstate->stat_tuple_bytes_recvd); - metrics->mutable_received()->set_chunks(mlstate->stat_total_chunks_recvd); - } - metrics->set_inherited_calls(nested_calls); - metrics->set_inherited_time(nested_time); +void +set_metric_instrumentation(gpsc::MetricInstrumentation *metrics, + QueryDesc *query_desc, int nested_calls, + double nested_time) +{ + auto instrument = query_desc->planstate->instrument; + if (instrument) + { + metrics->set_ntuples(instrument->ntuples); + metrics->set_nloops(instrument->nloops); + metrics->set_tuplecount(instrument->tuplecount); + metrics->set_firsttuple(instrument->firsttuple); + metrics->set_startup(instrument->startup); + metrics->set_total(instrument->total); + auto &buffusage = instrument->bufusage; + metrics->set_shared_blks_hit(buffusage.shared_blks_hit); + metrics->set_shared_blks_read(buffusage.shared_blks_read); + metrics->set_shared_blks_dirtied(buffusage.shared_blks_dirtied); + metrics->set_shared_blks_written(buffusage.shared_blks_written); + metrics->set_local_blks_hit(buffusage.local_blks_hit); + metrics->set_local_blks_read(buffusage.local_blks_read); + metrics->set_local_blks_dirtied(buffusage.local_blks_dirtied); + metrics->set_local_blks_written(buffusage.local_blks_written); + metrics->set_temp_blks_read(buffusage.temp_blks_read); + metrics->set_temp_blks_written(buffusage.temp_blks_written); + metrics->set_blk_read_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_read_time)); + metrics->set_blk_write_time( + INSTR_TIME_GET_DOUBLE(buffusage.blk_write_time)); + } + if (query_desc->estate && query_desc->estate->motionlayer_context) + { + MotionLayerState *mlstate = + (MotionLayerState *) query_desc->estate->motionlayer_context; + metrics->mutable_sent()->set_total_bytes( + mlstate->stat_total_bytes_sent); + metrics->mutable_sent()->set_tuple_bytes( + mlstate->stat_tuple_bytes_sent); + metrics->mutable_sent()->set_chunks(mlstate->stat_total_chunks_sent); + metrics->mutable_received()->set_total_bytes( + mlstate->stat_total_bytes_recvd); + metrics->mutable_received()->set_tuple_bytes( + mlstate->stat_tuple_bytes_recvd); + metrics->mutable_received()->set_chunks( + mlstate->stat_total_chunks_recvd); + } + metrics->set_inherited_calls(nested_calls); + metrics->set_inherited_time(nested_time); } -void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time) { - if (query_desc->planstate && query_desc->planstate->instrument) { - set_metric_instrumentation(metrics->mutable_instrumentation(), query_desc, - nested_calls, nested_time); - } - fill_self_stats(metrics->mutable_systemstat()); - metrics->mutable_systemstat()->set_runningtimeseconds( - time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); - metrics->mutable_spill()->set_filecount( - WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); - metrics->mutable_spill()->set_totalbytes( - WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); +void +set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, + int nested_calls, double nested_time) +{ + if (query_desc->planstate && query_desc->planstate->instrument) + { + set_metric_instrumentation(metrics->mutable_instrumentation(), + query_desc, nested_calls, nested_time); + } + fill_self_stats(metrics->mutable_systemstat()); + metrics->mutable_systemstat()->set_runningtimeseconds( + time(NULL) - metrics->mutable_systemstat()->runningtimeseconds()); + metrics->mutable_spill()->set_filecount( + WorkfileTotalFilesCreated() - metrics->mutable_spill()->filecount()); + metrics->mutable_spill()->set_totalbytes( + WorkfileTotalBytesWritten() - metrics->mutable_spill()->totalbytes()); } -#define UPDATE_IC_STATS(proto_name, stat_name) \ - metrics->mutable_interconnect()->set_##proto_name( \ - ic_statistics->stat_name - \ - metrics->mutable_interconnect()->proto_name()); \ - Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ - metrics->mutable_interconnect()->proto_name() <= \ - ic_statistics->stat_name) +#define UPDATE_IC_STATS(proto_name, stat_name) \ + metrics->mutable_interconnect()->set_##proto_name( \ + ic_statistics->stat_name - \ + metrics->mutable_interconnect()->proto_name()); \ + Assert(metrics->mutable_interconnect()->proto_name() >= 0 && \ + metrics->mutable_interconnect()->proto_name() <= \ + ic_statistics->stat_name) -void set_ic_stats(gpsc::MetricInstrumentation *metrics, - const ICStatistics *ic_statistics) { +void +set_ic_stats(gpsc::MetricInstrumentation *metrics, + const ICStatistics *ic_statistics) +{ #ifdef IC_TEARDOWN_HOOK - UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); - UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); - UPDATE_IC_STATS(total_capacity, totalCapacity); - UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); - UPDATE_IC_STATS(total_buffers, totalBuffers); - UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); - UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); - UPDATE_IC_STATS(retransmits, retransmits); - UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); - UPDATE_IC_STATS(mismatch_num, mismatchNum); - UPDATE_IC_STATS(crc_errors, crcErrors); - UPDATE_IC_STATS(snd_pkt_num, sndPktNum); - UPDATE_IC_STATS(recv_pkt_num, recvPktNum); - UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); - UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); - UPDATE_IC_STATS(recv_ack_num, recvAckNum); - UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); + UPDATE_IC_STATS(total_recv_queue_size, totalRecvQueueSize); + UPDATE_IC_STATS(recv_queue_size_counting_time, recvQueueSizeCountingTime); + UPDATE_IC_STATS(total_capacity, totalCapacity); + UPDATE_IC_STATS(capacity_counting_time, capacityCountingTime); + UPDATE_IC_STATS(total_buffers, totalBuffers); + UPDATE_IC_STATS(buffer_counting_time, bufferCountingTime); + UPDATE_IC_STATS(active_connections_num, activeConnectionsNum); + UPDATE_IC_STATS(retransmits, retransmits); + UPDATE_IC_STATS(startup_cached_pkt_num, startupCachedPktNum); + UPDATE_IC_STATS(mismatch_num, mismatchNum); + UPDATE_IC_STATS(crc_errors, crcErrors); + UPDATE_IC_STATS(snd_pkt_num, sndPktNum); + UPDATE_IC_STATS(recv_pkt_num, recvPktNum); + UPDATE_IC_STATS(disordered_pkt_num, disorderedPktNum); + UPDATE_IC_STATS(duplicated_pkt_num, duplicatedPktNum); + UPDATE_IC_STATS(recv_ack_num, recvAckNum); + UPDATE_IC_STATS(status_query_msg_num, statusQueryMsgNum); #endif } -gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status) { - gpsc::SetQueryReq req; - req.set_query_status(status); - *req.mutable_datetime() = current_ts(); - set_query_key(req.mutable_query_key()); - set_segment_key(req.mutable_segment_key()); - return req; +gpsc::SetQueryReq +create_query_req(gpsc::QueryStatus status) +{ + gpsc::SetQueryReq req; + req.set_query_status(status); + *req.mutable_datetime() = current_ts(); + set_query_key(req.mutable_query_key()); + set_segment_key(req.mutable_segment_key()); + return req; } -double protots_to_double(const google::protobuf::Timestamp &ts) { - return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; +double +protots_to_double(const google::protobuf::Timestamp &ts) +{ + return double(ts.seconds()) + double(ts.nanos()) / 1000000000.0; } -void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, - const Config &config) { - // Make sure it is a valid txn and it is not an utility - // statement for ExplainPrintPlan() later. - if (!IsTransactionState() || !query_desc->plannedstmt) { - return; - } - MemoryContext oldcxt = - gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); - ExplainState es = gpdb::get_analyze_state( - query_desc, query_desc->instrument_options && config.enable_analyze()); - gpdb::mem_ctx_switch_to(oldcxt); - if (es.str) { - // Remove last line break. - if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') { - es.str->data[--es.str->len] = '\0'; - } - auto trimmed_analyze = - trim_str_shrink_utf8(es.str->data, es.str->len, config.max_plan_size()); - req->mutable_query_info()->set_analyze_text(trimmed_analyze); - gpdb::pfree(es.str->data); - } +void +set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *req, + const Config &config) +{ + // Make sure it is a valid txn and it is not an utility + // statement for ExplainPrintPlan() later. + if (!IsTransactionState() || !query_desc->plannedstmt) + { + return; + } + MemoryContext oldcxt = + gpdb::mem_ctx_switch_to(query_desc->estate->es_query_cxt); + ExplainState es = gpdb::get_analyze_state( + query_desc, query_desc->instrument_options && config.enable_analyze()); + gpdb::mem_ctx_switch_to(oldcxt); + if (es.str) + { + // Remove last line break. + if (es.str->len > 0 && es.str->data[es.str->len - 1] == '\n') + { + es.str->data[--es.str->len] = '\0'; + } + auto trimmed_analyze = trim_str_shrink_utf8(es.str->data, es.str->len, + config.max_plan_size()); + req->mutable_query_info()->set_analyze_text(trimmed_analyze); + gpdb::pfree(es.str->data); + } } diff --git a/gpcontrib/gp_stats_collector/src/ProtoUtils.h b/gpcontrib/gp_stats_collector/src/ProtoUtils.h index 5ddcd42d308..6b38097fbcc 100644 --- a/gpcontrib/gp_stats_collector/src/ProtoUtils.h +++ b/gpcontrib/gp_stats_collector/src/ProtoUtils.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef PROTOUTILS_H +#define PROTOUTILS_H #include "protos/gpsc_set_service.pb.h" @@ -35,20 +36,22 @@ class Config; google::protobuf::Timestamp current_ts(); void set_query_plan(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config); + const Config &config); void set_query_text(gpsc::SetQueryReq *req, QueryDesc *query_desc, - const Config &config); + const Config &config); void clear_big_fields(gpsc::SetQueryReq *req); void set_query_info(gpsc::SetQueryReq *req); void set_qi_nesting_level(gpsc::SetQueryReq *req, int nesting_level); void set_qi_slice_id(gpsc::SetQueryReq *req); void set_qi_error_message(gpsc::SetQueryReq *req, const char *err_msg, - const Config &config); + const Config &config); void set_gp_metrics(gpsc::GPMetrics *metrics, QueryDesc *query_desc, - int nested_calls, double nested_time); + int nested_calls, double nested_time); void set_ic_stats(gpsc::MetricInstrumentation *metrics, - const ICStatistics *ic_statistics); + const ICStatistics *ic_statistics); gpsc::SetQueryReq create_query_req(gpsc::QueryStatus status); double protots_to_double(const google::protobuf::Timestamp &ts); void set_analyze_plan_text(QueryDesc *query_desc, gpsc::SetQueryReq *message, - const Config &config); + const Config &config); + +#endif /* PROTOUTILS_H */ diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp index 9a01d4033d0..16344366456 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -28,103 +28,119 @@ #include "UDSConnector.h" #include "Config.h" #include "GpscStat.h" -#include "memory/gpdbwrappers.h" #include "log/LogOps.h" +#include "memory/gpdbwrappers.h" +#include #include -#include +#include #include -#include #include -#include -#include +#include #include +#include extern "C" { #include "postgres.h" } static void inline log_tracing_failure(const gpsc::SetQueryReq &req, - const std::string &event) { - ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", - req.query_key().tmid(), req.query_key().ssid(), - req.query_key().ccnt(), event.c_str()))); + const std::string &event) +{ + ereport(LOG, (errmsg("Query {%d-%d-%d} %s tracing failed with error %m", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), event.c_str()))); } -bool UDSConnector::report_query(const gpsc::SetQueryReq &req, - const std::string &event, - const Config &config) { - sockaddr_un address{}; - address.sun_family = AF_UNIX; - const auto &uds_path = config.uds_path(); +bool +UDSConnector::report_query(const gpsc::SetQueryReq &req, + const std::string &event, const Config &config) +{ + sockaddr_un address{}; + address.sun_family = AF_UNIX; + const auto &uds_path = config.uds_path(); - if (uds_path.size() >= sizeof(address.sun_path)) { - ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); - GpscStat::report_error(); - return false; - } - strcpy(address.sun_path, uds_path.c_str()); + if (uds_path.size() >= sizeof(address.sun_path)) + { + ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); + GpscStat::report_error(); + return false; + } + strcpy(address.sun_path, uds_path.c_str()); - const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); - if (sockfd == -1) { - log_tracing_failure(req, event); - GpscStat::report_error(); - return false; - } + const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) + { + log_tracing_failure(req, event); + GpscStat::report_error(); + return false; + } - // Close socket automatically on error path. - struct SockGuard { - int fd; - ~SockGuard() { close(fd); } - } sock_guard{sockfd}; + // Close socket automatically on error path. + struct SockGuard + { + int fd; + ~SockGuard() + { + close(fd); + } + } sock_guard{sockfd}; - if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) { - // That's a very important error that should never happen, so make it - // visible to an end-user and admins. - ereport(WARNING, - (errmsg("Unable to create non-blocking socket connection %m"))); - GpscStat::report_error(); - return false; - } + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) + { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %m"))); + GpscStat::report_error(); + return false; + } - if (connect(sockfd, reinterpret_cast(&address), - sizeof(address)) == -1) { - log_tracing_failure(req, event); - GpscStat::report_bad_connection(); - return false; - } + if (connect(sockfd, reinterpret_cast(&address), + sizeof(address)) == -1) + { + log_tracing_failure(req, event); + GpscStat::report_bad_connection(); + return false; + } - const auto data_size = req.ByteSizeLong(); - const auto total_size = data_size + sizeof(uint32_t); - auto *buf = static_cast(gpdb::palloc(total_size)); - // Free buf automatically on error path. - struct BufGuard { - void *p; - ~BufGuard() { gpdb::pfree(p); } - } buf_guard{buf}; + const auto data_size = req.ByteSizeLong(); + const auto total_size = data_size + sizeof(uint32_t); + auto *buf = static_cast(gpdb::palloc(total_size)); + // Free buf automatically on error path. + struct BufGuard + { + void *p; + ~BufGuard() + { + gpdb::pfree(p); + } + } buf_guard{buf}; - *reinterpret_cast(buf) = data_size; - req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); + *reinterpret_cast(buf) = data_size; + req.SerializeWithCachedSizesToArray(buf + sizeof(uint32_t)); - int64_t sent = 0, sent_total = 0; - do { - sent = - send(sockfd, buf + sent_total, total_size - sent_total, MSG_DONTWAIT); - if (sent > 0) - sent_total += sent; - } while (sent > 0 && size_t(sent_total) != total_size && - // the line below is a small throttling hack: - // if a message does not fit a single packet, we take a nap - // before sending the next one. - // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + int64_t sent = 0, sent_total = 0; + do + { + sent = send(sockfd, buf + sent_total, total_size - sent_total, + MSG_DONTWAIT); + if (sent > 0) + sent_total += sent; + } while (sent > 0 && size_t(sent_total) != total_size && + // the line below is a small throttling hack: + // if a message does not fit a single packet, we take a nap + // before sending the next one. + // Otherwise, MSG_DONTWAIT send might overflow the UDS + (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); - if (sent < 0) { - log_tracing_failure(req, event); - GpscStat::report_bad_send(total_size); - return false; - } + if (sent < 0) + { + log_tracing_failure(req, event); + GpscStat::report_bad_send(total_size); + return false; + } - GpscStat::report_send(total_size); - return true; + GpscStat::report_send(total_size); + return true; } diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.h b/gpcontrib/gp_stats_collector/src/UDSConnector.h index a91d22f9df1..ac56dd54f44 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.h +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.h @@ -25,14 +25,18 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef UDSCONNECTOR_H +#define UDSCONNECTOR_H #include "protos/gpsc_set_service.pb.h" class Config; -class UDSConnector { +class UDSConnector +{ public: - bool static report_query(const gpsc::SetQueryReq &req, - const std::string &event, const Config &config); + bool static report_query(const gpsc::SetQueryReq &req, + const std::string &event, const Config &config); }; + +#endif /* UDSCONNECTOR_H */ diff --git a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c index d930f72246d..d295e37b396 100644 --- a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -45,106 +45,131 @@ PG_FUNCTION_INFO_V1(gpsc_test_uds_start_server); PG_FUNCTION_INFO_V1(gpsc_test_uds_receive); PG_FUNCTION_INFO_V1(gpsc_test_uds_stop_server); -void _PG_init(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) - hooks_init(); +void +_PG_init(void) +{ + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + hooks_init(); } -void _PG_fini(void) { - if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) - hooks_deinit(); +void +_PG_fini(void) +{ + if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + hooks_deinit(); } -Datum gpsc_stat_messages_reset(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_stat_messages_reset(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - gpsc_functions_reset(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + gpsc_functions_reset(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_stat_messages(PG_FUNCTION_ARGS) { - return gpsc_functions_get(fcinfo); +Datum +gpsc_stat_messages(PG_FUNCTION_ARGS) +{ + return gpsc_functions_get(fcinfo); } -Datum gpsc_init_log(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_init_log(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - init_log(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + init_log(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_truncate_log(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_truncate_log(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - truncate_log(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + truncate_log(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_start_server(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); - test_uds_start_server(path); - pfree(path); - } - - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); +Datum +gpsc_test_uds_start_server(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + test_uds_start_server(path); + pfree(path); + } + + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_receive(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; - int64 *result; +Datum +gpsc_test_uds_receive(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + int64 *result; - if (SRF_IS_FIRSTCALL()) { - MemoryContext oldcontext; + if (SRF_IS_FIRSTCALL()) + { + MemoryContext oldcontext; - funcctx = SRF_FIRSTCALL_INIT(); - oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); - result = (int64 *)palloc(sizeof(int64)); - funcctx->user_fctx = result; - funcctx->max_calls = 1; - MemoryContextSwitchTo(oldcontext); + funcctx = SRF_FIRSTCALL_INIT(); + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + result = (int64 *) palloc(sizeof(int64)); + funcctx->user_fctx = result; + funcctx->max_calls = 1; + MemoryContextSwitchTo(oldcontext); - int timeout_ms = PG_GETARG_INT32(0); - *result = test_uds_receive(timeout_ms); - } + int timeout_ms = PG_GETARG_INT32(0); + *result = test_uds_receive(timeout_ms); + } - funcctx = SRF_PERCALL_SETUP(); + funcctx = SRF_PERCALL_SETUP(); - if (funcctx->call_cntr < funcctx->max_calls) { - result = (int64 *)funcctx->user_fctx; - SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); - } + if (funcctx->call_cntr < funcctx->max_calls) + { + result = (int64 *) funcctx->user_fctx; + SRF_RETURN_NEXT(funcctx, Int64GetDatum(*result)); + } - SRF_RETURN_DONE(funcctx); + SRF_RETURN_DONE(funcctx); } -Datum gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) { - FuncCallContext *funcctx; +Datum +gpsc_test_uds_stop_server(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; - if (SRF_IS_FIRSTCALL()) { - funcctx = SRF_FIRSTCALL_INIT(); - test_uds_stop_server(); - } + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + test_uds_stop_server(); + } - funcctx = SRF_PERCALL_SETUP(); - SRF_RETURN_DONE(funcctx); + funcctx = SRF_PERCALL_SETUP(); + SRF_RETURN_DONE(funcctx); } diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index 0a40b4cb359..3f19d4d9930 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -28,28 +28,28 @@ #define typeid __typeid extern "C" { #include "postgres.h" -#include "funcapi.h" -#include "executor/executor.h" -#include "executor/execUtils.h" -#include "utils/elog.h" -#include "utils/builtins.h" -#include "utils/metrics_utils.h" #include "cdb/cdbvars.h" #include "cdb/ml_ipc.h" +#include "executor/execUtils.h" +#include "executor/executor.h" +#include "funcapi.h" +#include "stat_statements_parser/pg_stat_statements_parser.h" #include "tcop/utility.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "utils/metrics_utils.h" +#include +#include #include #include #include -#include -#include } #undef typeid #include "Config.h" -#include "GpscStat.h" #include "EventSender.h" +#include "GpscStat.h" #include "hook_wrappers.h" #include "memory/gpdbwrappers.h" @@ -60,7 +60,7 @@ static ExecutorEnd_hook_type previous_ExecutorEnd_hook = nullptr; static query_info_collect_hook_type previous_query_info_collect_hook = nullptr; #ifdef ANALYZE_STATS_COLLECT_HOOK static analyze_stats_collect_hook_type previous_analyze_stats_collect_hook = - nullptr; + nullptr; #endif #ifdef IC_TEARDOWN_HOOK static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; @@ -68,24 +68,23 @@ static ic_teardown_hook_type previous_ic_teardown_hook = nullptr; static ProcessUtility_hook_type previous_ProcessUtility_hook = nullptr; static void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags); -static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - uint64 count, bool execute_once); +static void gpsc_ExecutorRun_hook(QueryDesc *query_desc, + ScanDirection direction, uint64 count, + bool execute_once); static void gpsc_ExecutorFinish_hook(QueryDesc *query_desc); static void gpsc_ExecutorEnd_hook(QueryDesc *query_desc); static void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg); #ifdef IC_TEARDOWN_HOOK static void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, - bool hasErrors); + bool hasErrors); #endif #ifdef ANALYZE_STATS_COLLECT_HOOK static void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc); #endif -static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, - bool readOnlyTree, - ProcessUtilityContext context, - ParamListInfo params, - QueryEnvironment *queryEnv, - DestReceiver *dest, QueryCompletion *qc); +static void gpsc_process_utility_hook( + PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, + ProcessUtilityContext context, ParamListInfo params, + QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); #define TEST_MAX_CONNECTIONS 4 #define TEST_RCV_BUF_SIZE 8192 @@ -96,319 +95,379 @@ static char *test_sock_path = NULL; static EventSender *sender = nullptr; -static inline EventSender *get_sender() { - if (!sender) { - sender = new EventSender(); - } - return sender; +static inline EventSender * +get_sender() +{ + if (!sender) + { + sender = new EventSender(); + } + return sender; } template -R cpp_call(T *obj, R (T::*func)(Args...), Args... args) { - try { - return (obj->*func)(args...); - } catch (const std::exception &e) { - ereport(FATAL, (errmsg("Unexpected exception in gpsc %s", e.what()))); - } +R +cpp_call(T *obj, R (T::*func)(Args...), Args... args) +{ + try + { + return (obj->*func)(args...); + } + catch (const std::exception &e) + { + ereport(ERROR, (errmsg("Unexpected exception in gpsc %s", e.what()))); + } } -void hooks_init() { - Config::init_gucs(); - GpscStat::init(); - previous_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = gpsc_ExecutorStart_hook; - previous_ExecutorRun_hook = ExecutorRun_hook; - ExecutorRun_hook = gpsc_ExecutorRun_hook; - previous_ExecutorFinish_hook = ExecutorFinish_hook; - ExecutorFinish_hook = gpsc_ExecutorFinish_hook; - previous_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = gpsc_ExecutorEnd_hook; - previous_query_info_collect_hook = query_info_collect_hook; - query_info_collect_hook = gpsc_query_info_collect_hook; +void +hooks_init() +{ + Config::init_gucs(); + GpscStat::init(); + previous_ExecutorStart_hook = ExecutorStart_hook; + ExecutorStart_hook = gpsc_ExecutorStart_hook; + previous_ExecutorRun_hook = ExecutorRun_hook; + ExecutorRun_hook = gpsc_ExecutorRun_hook; + previous_ExecutorFinish_hook = ExecutorFinish_hook; + ExecutorFinish_hook = gpsc_ExecutorFinish_hook; + previous_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = gpsc_ExecutorEnd_hook; + previous_query_info_collect_hook = query_info_collect_hook; + query_info_collect_hook = gpsc_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK - previous_ic_teardown_hook = ic_teardown_hook; - ic_teardown_hook = gpsc_ic_teardown_hook; + previous_ic_teardown_hook = ic_teardown_hook; + ic_teardown_hook = gpsc_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK - previous_analyze_stats_collect_hook = analyze_stats_collect_hook; - analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; + previous_analyze_stats_collect_hook = analyze_stats_collect_hook; + analyze_stats_collect_hook = gpsc_analyze_stats_collect_hook; #endif - stat_statements_parser_init(); - previous_ProcessUtility_hook = ProcessUtility_hook; - ProcessUtility_hook = gpsc_process_utility_hook; + stat_statements_parser_init(); + previous_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = gpsc_process_utility_hook; } -void hooks_deinit() { - ExecutorStart_hook = previous_ExecutorStart_hook; - ExecutorEnd_hook = previous_ExecutorEnd_hook; - ExecutorRun_hook = previous_ExecutorRun_hook; - ExecutorFinish_hook = previous_ExecutorFinish_hook; - query_info_collect_hook = previous_query_info_collect_hook; +void +hooks_deinit() +{ + ExecutorStart_hook = previous_ExecutorStart_hook; + ExecutorEnd_hook = previous_ExecutorEnd_hook; + ExecutorRun_hook = previous_ExecutorRun_hook; + ExecutorFinish_hook = previous_ExecutorFinish_hook; + query_info_collect_hook = previous_query_info_collect_hook; #ifdef IC_TEARDOWN_HOOK - ic_teardown_hook = previous_ic_teardown_hook; + ic_teardown_hook = previous_ic_teardown_hook; #endif #ifdef ANALYZE_STATS_COLLECT_HOOK - analyze_stats_collect_hook = previous_analyze_stats_collect_hook; + analyze_stats_collect_hook = previous_analyze_stats_collect_hook; #endif - stat_statements_parser_deinit(); - if (sender) { - delete sender; - } - GpscStat::deinit(); - ProcessUtility_hook = previous_ProcessUtility_hook; + stat_statements_parser_deinit(); + if (sender) + { + delete sender; + } + GpscStat::deinit(); + ProcessUtility_hook = previous_ProcessUtility_hook; } -void gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) { - cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, - eflags); - if (previous_ExecutorStart_hook) { - (*previous_ExecutorStart_hook)(query_desc, eflags); - } else { - standard_ExecutorStart(query_desc, eflags); - } - cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, - eflags); +void +gpsc_ExecutorStart_hook(QueryDesc *query_desc, int eflags) +{ + cpp_call(get_sender(), &EventSender::executor_before_start, query_desc, + eflags); + if (previous_ExecutorStart_hook) + { + (*previous_ExecutorStart_hook)(query_desc, eflags); + } + else + { + standard_ExecutorStart(query_desc, eflags); + } + cpp_call(get_sender(), &EventSender::executor_after_start, query_desc, + eflags); } -void gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, - uint64 count, bool execute_once) { - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ExecutorRun_hook) - previous_ExecutorRun_hook(query_desc, direction, count, execute_once); - else - standard_ExecutorRun(query_desc, direction, count, execute_once); - get_sender()->decr_depth(); - } - PG_CATCH(); - { - get_sender()->decr_depth(); - PG_RE_THROW(); - } - PG_END_TRY(); +void +gpsc_ExecutorRun_hook(QueryDesc *query_desc, ScanDirection direction, + uint64 count, bool execute_once) +{ + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorRun_hook) + previous_ExecutorRun_hook(query_desc, direction, count, + execute_once); + else + standard_ExecutorRun(query_desc, direction, count, execute_once); + get_sender()->decr_depth(); + } + PG_CATCH(); + { + get_sender()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); } -void gpsc_ExecutorFinish_hook(QueryDesc *query_desc) { - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ExecutorFinish_hook) - previous_ExecutorFinish_hook(query_desc); - else - standard_ExecutorFinish(query_desc); - get_sender()->decr_depth(); - } - PG_CATCH(); - { - get_sender()->decr_depth(); - PG_RE_THROW(); - } - PG_END_TRY(); +void +gpsc_ExecutorFinish_hook(QueryDesc *query_desc) +{ + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ExecutorFinish_hook) + previous_ExecutorFinish_hook(query_desc); + else + standard_ExecutorFinish(query_desc); + get_sender()->decr_depth(); + } + PG_CATCH(); + { + get_sender()->decr_depth(); + PG_RE_THROW(); + } + PG_END_TRY(); } -void gpsc_ExecutorEnd_hook(QueryDesc *query_desc) { - cpp_call(get_sender(), &EventSender::executor_end, query_desc); - if (previous_ExecutorEnd_hook) { - (*previous_ExecutorEnd_hook)(query_desc); - } else { - standard_ExecutorEnd(query_desc); - } +void +gpsc_ExecutorEnd_hook(QueryDesc *query_desc) +{ + cpp_call(get_sender(), &EventSender::executor_end, query_desc); + if (previous_ExecutorEnd_hook) + { + (*previous_ExecutorEnd_hook)(query_desc); + } + else + { + standard_ExecutorEnd(query_desc); + } } -void gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) { - cpp_call(get_sender(), &EventSender::query_metrics_collect, status, - arg /* queryDesc */, false /* utility */, (ErrorData *)NULL); - if (previous_query_info_collect_hook) { - (*previous_query_info_collect_hook)(status, arg); - } +void +gpsc_query_info_collect_hook(QueryMetricsStatus status, void *arg) +{ + cpp_call(get_sender(), &EventSender::query_metrics_collect, status, + arg /* queryDesc */, false /* utility */, (ErrorData *) NULL); + if (previous_query_info_collect_hook) + { + (*previous_query_info_collect_hook)(status, arg); + } } #ifdef IC_TEARDOWN_HOOK -void gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) { - cpp_call(get_sender(), &EventSender::ic_metrics_collect); - if (previous_ic_teardown_hook) { - (*previous_ic_teardown_hook)(transportStates, hasErrors); - } +void +gpsc_ic_teardown_hook(ChunkTransportState *transportStates, bool hasErrors) +{ + cpp_call(get_sender(), &EventSender::ic_metrics_collect); + if (previous_ic_teardown_hook) + { + (*previous_ic_teardown_hook)(transportStates, hasErrors); + } } #endif #ifdef ANALYZE_STATS_COLLECT_HOOK -void gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) { - cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); - if (previous_analyze_stats_collect_hook) { - (*previous_analyze_stats_collect_hook)(query_desc); - } +void +gpsc_analyze_stats_collect_hook(QueryDesc *query_desc) +{ + cpp_call(get_sender(), &EventSender::analyze_stats_collect, query_desc); + if (previous_analyze_stats_collect_hook) + { + (*previous_analyze_stats_collect_hook)(query_desc); + } } #endif -static void gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, - bool readOnlyTree, - ProcessUtilityContext context, - ParamListInfo params, - QueryEnvironment *queryEnv, - DestReceiver *dest, QueryCompletion *qc) { - /* Project utility data on QueryDesc to use existing logic */ - QueryDesc *query_desc = (QueryDesc *)palloc0(sizeof(QueryDesc)); - query_desc->sourceText = queryString; - - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_SUBMIT, (void *)query_desc, true /* utility */, - (ErrorData *)NULL); - - get_sender()->incr_depth(); - PG_TRY(); - { - if (previous_ProcessUtility_hook) { - (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, - params, queryEnv, dest, qc); - } else { - standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, - queryEnv, dest, qc); - } - - get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_DONE, (void *)query_desc, true /* utility */, - (ErrorData *)NULL); - - pfree(query_desc); - } - PG_CATCH(); - { - ErrorData *edata; - MemoryContext oldctx; - - oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - FlushErrorState(); - MemoryContextSwitchTo(oldctx); - - get_sender()->decr_depth(); - cpp_call(get_sender(), &EventSender::query_metrics_collect, - METRICS_QUERY_ERROR, (void *)query_desc, true /* utility */, - edata); - - pfree(query_desc); - ReThrowError(edata); - } - PG_END_TRY(); +static void +gpsc_process_utility_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, ProcessUtilityContext context, + ParamListInfo params, QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) +{ + /* Project utility data on QueryDesc to use existing logic */ + QueryDesc *query_desc = (QueryDesc *) palloc0(sizeof(QueryDesc)); + query_desc->sourceText = queryString; + + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_SUBMIT, (void *) query_desc, true /* utility */, + (ErrorData *) NULL); + + get_sender()->incr_depth(); + PG_TRY(); + { + if (previous_ProcessUtility_hook) + { + (*previous_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, + qc); + } + else + { + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); + } + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_DONE, (void *) query_desc, true /* utility */, + (ErrorData *) NULL); + + pfree(query_desc); + } + PG_CATCH(); + { + ErrorData *edata; + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + FlushErrorState(); + MemoryContextSwitchTo(oldctx); + + get_sender()->decr_depth(); + cpp_call(get_sender(), &EventSender::query_metrics_collect, + METRICS_QUERY_ERROR, (void *) query_desc, true /* utility */, + edata); + + pfree(query_desc); + ReThrowError(edata); + } + PG_END_TRY(); } -static void check_stats_loaded() { - if (!GpscStat::loaded()) { - ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("gp_stats_collector must be loaded via " - "shared_preload_libraries"))); - } +static void +check_stats_loaded() +{ + if (!GpscStat::loaded()) + { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("gp_stats_collector must be loaded via " + "shared_preload_libraries"))); + } } -void gpsc_functions_reset() { - check_stats_loaded(); - GpscStat::reset(); +void +gpsc_functions_reset() +{ + check_stats_loaded(); + GpscStat::reset(); } -Datum gpsc_functions_get(FunctionCallInfo fcinfo) { - const int ATTNUM = 6; - check_stats_loaded(); - auto stats = GpscStat::get_stats(); - TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); - TupleDescInitEntry(tupdesc, (AttrNumber)1, "segid", INT4OID, -1 /* typmod */, - 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)2, "total_messages", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)3, "send_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)4, "connection_failures", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)5, "other_errors", INT8OID, - -1 /* typmod */, 0 /* attdim */); - TupleDescInitEntry(tupdesc, (AttrNumber)6, "max_message_size", INT4OID, - -1 /* typmod */, 0 /* attdim */); - tupdesc = BlessTupleDesc(tupdesc); - Datum values[ATTNUM]; - bool nulls[ATTNUM]; - MemSet(nulls, 0, sizeof(nulls)); - values[0] = Int32GetDatum(GpIdentity.segindex); - values[1] = Int64GetDatum(stats.total); - values[2] = Int64GetDatum(stats.failed_sends); - values[3] = Int64GetDatum(stats.failed_connects); - values[4] = Int64GetDatum(stats.failed_other); - values[5] = Int32GetDatum(stats.max_message_size); - HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); - Datum result = HeapTupleGetDatum(tuple); - PG_RETURN_DATUM(result); +Datum +gpsc_functions_get(FunctionCallInfo fcinfo) +{ + const int ATTNUM = 6; + check_stats_loaded(); + auto stats = GpscStat::get_stats(); + TupleDesc tupdesc = CreateTemplateTupleDesc(ATTNUM); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segid", INT4OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "total_messages", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "send_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "connection_failures", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "other_errors", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "max_message_size", INT4OID, + -1 /* typmod */, 0 /* attdim */); + tupdesc = BlessTupleDesc(tupdesc); + Datum values[ATTNUM]; + bool nulls[ATTNUM]; + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(GpIdentity.segindex); + values[1] = Int64GetDatum(stats.total); + values[2] = Int64GetDatum(stats.failed_sends); + values[3] = Int64GetDatum(stats.failed_connects); + values[4] = Int64GetDatum(stats.failed_other); + values[5] = Int32GetDatum(stats.max_message_size); + HeapTuple tuple = gpdb::heap_form_tuple(tupdesc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + PG_RETURN_DATUM(result); } -void test_uds_stop_server() { - if (test_server_fd >= 0) { - close(test_server_fd); - test_server_fd = -1; - } - if (test_sock_path) { - unlink(test_sock_path); - pfree(test_sock_path); - test_sock_path = NULL; - } +void +test_uds_stop_server() +{ + if (test_server_fd >= 0) + { + close(test_server_fd); + test_server_fd = -1; + } + if (test_sock_path) + { + unlink(test_sock_path); + pfree(test_sock_path); + test_sock_path = NULL; + } } -void test_uds_start_server(const char *path) { - struct sockaddr_un addr = {.sun_family = AF_UNIX}; +void +test_uds_start_server(const char *path) +{ + struct sockaddr_un addr = {.sun_family = AF_UNIX}; - if (strlen(path) >= sizeof(addr.sun_path)) - ereport(ERROR, (errmsg("path too long"))); + if (strlen(path) >= sizeof(addr.sun_path)) + ereport(ERROR, (errmsg("path too long"))); - test_uds_stop_server(); + test_uds_stop_server(); - strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); - test_sock_path = MemoryContextStrdup(TopMemoryContext, path); - unlink(path); + strlcpy(addr.sun_path, path, sizeof(addr.sun_path)); + test_sock_path = MemoryContextStrdup(TopMemoryContext, path); + unlink(path); - if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || - bind(test_server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0 || - listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) { - test_uds_stop_server(); - ereport(ERROR, (errmsg("socket setup failed: %m"))); - } + if ((test_server_fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 || + bind(test_server_fd, (struct sockaddr *) &addr, sizeof(addr)) < 0 || + listen(test_server_fd, TEST_MAX_CONNECTIONS) < 0) + { + test_uds_stop_server(); + ereport(ERROR, (errmsg("socket setup failed: %m"))); + } } -int64 test_uds_receive(int timeout_ms) { - char buf[TEST_RCV_BUF_SIZE]; - int rc; - struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; - int64 total = 0; - - if (test_server_fd < 0) - ereport(ERROR, (errmsg("server not started"))); - - for (;;) { - CHECK_FOR_INTERRUPTS(); - rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); - if (rc > 0) - break; - if (rc < 0 && errno != EINTR) - ereport(ERROR, (errmsg("poll: %m"))); - timeout_ms -= TEST_POLL_TIMEOUT_MS; - if (timeout_ms <= 0) - return total; - } - - if (pfd.revents & POLLIN) { - int client = accept(test_server_fd, NULL, NULL); - ssize_t n; - - if (client < 0) - ereport(ERROR, (errmsg("accept: %m"))); - - while ((n = recv(client, buf, sizeof(buf), 0)) != 0) { - if (n > 0) - total += n; - else if (errno != EINTR) - break; - } - - close(client); - } - - return total; +int64 +test_uds_receive(int timeout_ms) +{ + char buf[TEST_RCV_BUF_SIZE]; + int rc; + struct pollfd pfd = {.fd = test_server_fd, .events = POLLIN}; + int64 total = 0; + + if (test_server_fd < 0) + ereport(ERROR, (errmsg("server not started"))); + + for (;;) + { + CHECK_FOR_INTERRUPTS(); + rc = poll(&pfd, 1, Min(timeout_ms, TEST_POLL_TIMEOUT_MS)); + if (rc > 0) + break; + if (rc < 0 && errno != EINTR) + ereport(ERROR, (errmsg("poll: %m"))); + timeout_ms -= TEST_POLL_TIMEOUT_MS; + if (timeout_ms <= 0) + return total; + } + + if (pfd.revents & POLLIN) + { + int client = accept(test_server_fd, NULL, NULL); + ssize_t n; + + if (client < 0) + ereport(ERROR, (errmsg("accept: %m"))); + + while ((n = recv(client, buf, sizeof(buf), 0)) != 0) + { + if (n > 0) + total += n; + else if (errno != EINTR) + break; + } + + close(client); + } + + return total; } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.h b/gpcontrib/gp_stats_collector/src/hook_wrappers.h index 06c8d064404..a04f5a95144 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.h +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef HOOK_WRAPPERS_H +#define HOOK_WRAPPERS_H #ifdef __cplusplus extern "C" { @@ -45,4 +46,5 @@ extern void test_uds_stop_server(); #ifdef __cplusplus } -#endif \ No newline at end of file +#endif +#endif /* HOOK_WRAPPERS_H */ diff --git a/gpcontrib/gp_stats_collector/src/log/LogOps.cpp b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp index ef4f39c0749..865e0f6ce3f 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogOps.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.cpp @@ -43,8 +43,8 @@ extern "C" { #include "catalog/pg_type.h" #include "cdb/cdbvars.h" #include "commands/tablecmds.h" -#include "funcapi.h" #include "fmgr.h" +#include "funcapi.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/rel.h" @@ -52,107 +52,122 @@ extern "C" { #include "utils/timestamp.h" } -void init_log() { - Oid namespaceId; - Oid relationId; - ObjectAddress tableAddr; - ObjectAddress schemaAddr; - - namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); - - /* Create table */ - relationId = heap_create_with_catalog( - log_relname.data() /* relname */, namespaceId /* namespace */, - 0 /* tablespace */, InvalidOid /* relid */, InvalidOid /* reltype oid */, - InvalidOid /* reloftypeid */, GetUserId() /* owner */, HEAP_TABLE_AM_OID, - DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, RELKIND_RELATION, - RELPERSISTENCE_PERMANENT, false /* shared_relation */, false /* mapped_relation */, ONCOMMIT_NOOP, - NULL /* GP Policy */, (Datum)0 /* reloptions */, false /* use_user_acl */, true /* allow_system_table_mods */, true /* is_internal */, - InvalidOid /* relrewrite */, NULL /* typaddress */, - false /* valid_opts */); - - /* Make the table visible */ - CommandCounterIncrement(); - - /* Record dependency of the table on the schema */ - if (OidIsValid(relationId) && OidIsValid(namespaceId)) { - ObjectAddressSet(tableAddr, RelationRelationId, relationId); - ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); - - /* Table can be dropped only via DROP EXTENSION */ - recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); - } else { - ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); - } - - /* Make changes visible */ - CommandCounterIncrement(); +void +init_log() +{ + Oid namespaceId; + Oid relationId; + ObjectAddress tableAddr; + ObjectAddress schemaAddr; + + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + + /* Create table */ + relationId = heap_create_with_catalog( + log_relname.data() /* relname */, namespaceId /* namespace */, + 0 /* tablespace */, InvalidOid /* relid */, + InvalidOid /* reltype oid */, InvalidOid /* reloftypeid */, + GetUserId() /* owner */, HEAP_TABLE_AM_OID, + DescribeTuple() /* rel tuple */, NIL /* cooked_constraints */, + RELKIND_RELATION, RELPERSISTENCE_PERMANENT, false /* shared_relation */, + false /* mapped_relation */, ONCOMMIT_NOOP, NULL /* GP Policy */, + (Datum) 0 /* reloptions */, false /* use_user_acl */, + true /* allow_system_table_mods */, true /* is_internal */, + InvalidOid /* relrewrite */, NULL /* typaddress */, + false /* valid_opts */); + + /* Make the table visible */ + CommandCounterIncrement(); + + /* Record dependency of the table on the schema */ + if (OidIsValid(relationId) && OidIsValid(namespaceId)) + { + ObjectAddressSet(tableAddr, RelationRelationId, relationId); + ObjectAddressSet(schemaAddr, NamespaceRelationId, namespaceId); + + /* Table can be dropped only via DROP EXTENSION */ + recordDependencyOn(&tableAddr, &schemaAddr, DEPENDENCY_EXTENSION); + } + else + { + ereport(NOTICE, (errmsg("GPSC failed to create log table or schema"))); + } + + /* Make changes visible */ + CommandCounterIncrement(); } -void insert_log(const gpsc::SetQueryReq &req, bool utility) { - Oid namespaceId; - Oid relationId; - Relation rel; - HeapTuple tuple; - - /* Return if xact is not valid (needed for catalog lookups). */ - if (!IsTransactionState()) { - return; - } - - /* Return if extension was not loaded */ - namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); - if (!OidIsValid(namespaceId)) { - return; - } - - /* Return if the table was not created yet */ - relationId = get_relname_relid(log_relname.data(), namespaceId); - if (!OidIsValid(relationId)) { - return; - } - - bool nulls[natts_gpsc_log]; - Datum values[natts_gpsc_log]; - - memset(nulls, true, sizeof(nulls)); - memset(values, 0, sizeof(values)); - - extract_query_req(req, "", values, nulls); - nulls[attnum_gpsc_log_utility] = false; - values[attnum_gpsc_log_utility] = BoolGetDatum(utility); - - rel = heap_open(relationId, RowExclusiveLock); - - /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls +void +insert_log(const gpsc::SetQueryReq &req, bool utility) +{ + Oid namespaceId; + Oid relationId; + Relation rel; + HeapTuple tuple; + + /* Return if xact is not valid (needed for catalog lookups). */ + if (!IsTransactionState()) + { + return; + } + + /* Return if extension was not loaded */ + namespaceId = get_namespace_oid(schema_name.data(), true /* missing_ok */); + if (!OidIsValid(namespaceId)) + { + return; + } + + /* Return if the table was not created yet */ + relationId = get_relname_relid(log_relname.data(), namespaceId); + if (!OidIsValid(relationId)) + { + return; + } + + bool nulls[natts_gpsc_log]; + Datum values[natts_gpsc_log]; + + memset(nulls, true, sizeof(nulls)); + memset(values, 0, sizeof(values)); + + extract_query_req(req, "", values, nulls); + nulls[attnum_gpsc_log_utility] = false; + values[attnum_gpsc_log_utility] = BoolGetDatum(utility); + + rel = heap_open(relationId, RowExclusiveLock); + + /* Insert the tuple as a frozen one to ensure it is logged even if txn rolls * back or aborts */ - tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); - frozen_heap_insert(rel, tuple); + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + frozen_heap_insert(rel, tuple); - heap_freetuple(tuple); - /* Keep lock on rel until end of xact */ - heap_close(rel, NoLock); + heap_freetuple(tuple); + /* Keep lock on rel until end of xact */ + heap_close(rel, NoLock); - /* Make changes visible */ - CommandCounterIncrement(); + /* Make changes visible */ + CommandCounterIncrement(); } -void truncate_log() { - Oid namespaceId; - Oid relationId; - Relation relation; +void +truncate_log() +{ + Oid namespaceId; + Oid relationId; + Relation relation; - namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); - relationId = get_relname_relid(log_relname.data(), namespaceId); + namespaceId = get_namespace_oid(schema_name.data(), false /* missing_ok */); + relationId = get_relname_relid(log_relname.data(), namespaceId); - relation = heap_open(relationId, AccessExclusiveLock); + relation = heap_open(relationId, AccessExclusiveLock); - /* Truncate the main table */ - heap_truncate_one_rel(relation); + /* Truncate the main table */ + heap_truncate_one_rel(relation); - /* Keep lock on rel until end of xact */ - heap_close(relation, NoLock); + /* Keep lock on rel until end of xact */ + heap_close(relation, NoLock); - /* Make changes visible */ - CommandCounterIncrement(); + /* Make changes visible */ + CommandCounterIncrement(); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/log/LogOps.h b/gpcontrib/gp_stats_collector/src/log/LogOps.h index f784270bb8f..45d79cd4560 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogOps.h +++ b/gpcontrib/gp_stats_collector/src/log/LogOps.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef LOGOPS_H +#define LOGOPS_H #include @@ -44,3 +45,5 @@ void truncate_log(); /* INSERT INTO gpsc.__log VALUES (...) */ void insert_log(const gpsc::SetQueryReq &req, bool utility); + +#endif /* LOGOPS_H */ diff --git a/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp index f9f43fac2fd..254b1b04af4 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.cpp @@ -25,138 +25,165 @@ *------------------------------------------------------------------------- */ -#include "google/protobuf/reflection.h" #include "google/protobuf/descriptor.h" +#include "google/protobuf/reflection.h" #include "google/protobuf/timestamp.pb.h" #include "LogSchema.h" -const std::unordered_map &proto_name_to_col_idx() { - static const auto name_col_idx = [] { - std::unordered_map map; - map.reserve(log_tbl_desc.size()); - - for (size_t idx = 0; idx < natts_gpsc_log; ++idx) { - map.emplace(log_tbl_desc[idx].proto_field_name, idx); - } - - return map; - }(); - return name_col_idx; +const std::unordered_map & +proto_name_to_col_idx() +{ + static const auto name_col_idx = [] { + std::unordered_map map; + map.reserve(log_tbl_desc.size()); + + for (size_t idx = 0; idx < natts_gpsc_log; ++idx) + { + map.emplace(log_tbl_desc[idx].proto_field_name, idx); + } + + return map; + }(); + return name_col_idx; } -TupleDesc DescribeTuple() { - TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); +TupleDesc +DescribeTuple() +{ + TupleDesc tupdesc = CreateTemplateTupleDesc(natts_gpsc_log); - for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) { - TupleDescInitEntry(tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), - log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, - 0 /* attdim */); - } + for (size_t anum = 1; anum <= natts_gpsc_log; ++anum) + { + TupleDescInitEntry( + tupdesc, anum, log_tbl_desc[anum - 1].pg_att_name.data(), + log_tbl_desc[anum - 1].type_oid, -1 /* typmod */, 0 /* attdim */); + } - return tupdesc; + return tupdesc; } -Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts) { - TimestampTz pgtimestamp = - (TimestampTz)ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); - pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; - return TimestampTzGetDatum(pgtimestamp); +Datum +protots_to_timestamptz(const google::protobuf::Timestamp &ts) +{ + TimestampTz pgtimestamp = + (TimestampTz) ts.seconds() * USECS_PER_SEC + (ts.nanos() / 1000); + pgtimestamp -= (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY; + return TimestampTzGetDatum(pgtimestamp); } -Datum field_to_datum(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg) { - using namespace google::protobuf; - - switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - return Int32GetDatum(reflection->GetInt32(msg, field)); - case FieldDescriptor::CPPTYPE_INT64: - return Int64GetDatum(reflection->GetInt64(msg, field)); - case FieldDescriptor::CPPTYPE_UINT32: - return Int64GetDatum(reflection->GetUInt32(msg, field)); - case FieldDescriptor::CPPTYPE_UINT64: - return Int64GetDatum( - static_cast(reflection->GetUInt64(msg, field))); - case FieldDescriptor::CPPTYPE_DOUBLE: - return Float8GetDatum(reflection->GetDouble(msg, field)); - case FieldDescriptor::CPPTYPE_FLOAT: - return Float4GetDatum(reflection->GetFloat(msg, field)); - case FieldDescriptor::CPPTYPE_BOOL: - return BoolGetDatum(reflection->GetBool(msg, field)); - case FieldDescriptor::CPPTYPE_ENUM: - return CStringGetTextDatum(reflection->GetEnum(msg, field)->name().data()); - case FieldDescriptor::CPPTYPE_STRING: - return CStringGetTextDatum(reflection->GetString(msg, field).c_str()); - default: - return (Datum)0; - } +Datum +field_to_datum(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg) +{ + using namespace google::protobuf; + + switch (field->cpp_type()) + { + case FieldDescriptor::CPPTYPE_INT32: + return Int32GetDatum(reflection->GetInt32(msg, field)); + case FieldDescriptor::CPPTYPE_INT64: + return Int64GetDatum(reflection->GetInt64(msg, field)); + case FieldDescriptor::CPPTYPE_UINT32: + return Int64GetDatum(reflection->GetUInt32(msg, field)); + case FieldDescriptor::CPPTYPE_UINT64: + return Int64GetDatum( + static_cast(reflection->GetUInt64(msg, field))); + case FieldDescriptor::CPPTYPE_DOUBLE: + return Float8GetDatum(reflection->GetDouble(msg, field)); + case FieldDescriptor::CPPTYPE_FLOAT: + return Float4GetDatum(reflection->GetFloat(msg, field)); + case FieldDescriptor::CPPTYPE_BOOL: + return BoolGetDatum(reflection->GetBool(msg, field)); + case FieldDescriptor::CPPTYPE_ENUM: + return CStringGetTextDatum( + reflection->GetEnum(msg, field)->name().data()); + case FieldDescriptor::CPPTYPE_STRING: + return CStringGetTextDatum( + reflection->GetString(msg, field).c_str()); + default: + return (Datum) 0; + } } -void process_field(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg, - const std::string &field_name, Datum *values, bool *nulls) { - - auto proto_idx_map = proto_name_to_col_idx(); - auto it = proto_idx_map.find(field_name); - - if (it == proto_idx_map.end()) { - ereport(NOTICE, - (errmsg("GPSC protobuf field %s is not registered in log table", - field_name.c_str()))); - return; - } - - int idx = it->second; - - if (!reflection->HasField(msg, field)) { - nulls[idx] = true; - return; - } - - if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && - field->message_type()->full_name() == "google.protobuf.Timestamp") { - const auto &ts = static_cast( - reflection->GetMessage(msg, field)); - values[idx] = protots_to_timestamptz(ts); - } else { - values[idx] = field_to_datum(field, reflection, msg); - } - nulls[idx] = false; - - return; +void +process_field(const google::protobuf::FieldDescriptor *field, + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls) +{ + auto proto_idx_map = proto_name_to_col_idx(); + auto it = proto_idx_map.find(field_name); + + if (it == proto_idx_map.end()) + { + ereport(NOTICE, + (errmsg("GPSC protobuf field %s is not registered in log table", + field_name.c_str()))); + return; + } + + int idx = it->second; + + if (!reflection->HasField(msg, field)) + { + nulls[idx] = true; + return; + } + + if (field->cpp_type() == + google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() == "google.protobuf.Timestamp") + { + const auto &ts = static_cast( + reflection->GetMessage(msg, field)); + values[idx] = protots_to_timestamptz(ts); + } + else + { + values[idx] = field_to_datum(field, reflection, msg); + } + nulls[idx] = false; + + return; } -void extract_query_req(const google::protobuf::Message &msg, - const std::string &prefix, Datum *values, bool *nulls) { - using namespace google::protobuf; - - const Descriptor *descriptor = msg.GetDescriptor(); - const Reflection *reflection = msg.GetReflection(); - - for (int i = 0; i < descriptor->field_count(); ++i) { - const FieldDescriptor *field = descriptor->field(i); - - // For now, we do not log any repeated fields plus they need special - // treatment. - if (field->is_repeated()) { - continue; - } - - std::string curr_pref = prefix.empty() ? "" : prefix + "."; - std::string field_name = curr_pref + field->name().data(); - - if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && - field->message_type()->full_name() != "google.protobuf.Timestamp") { - - if (reflection->HasField(msg, field)) { - const Message &nested = reflection->GetMessage(msg, field); - extract_query_req(nested, field_name, values, nulls); - } - } else { - process_field(field, reflection, msg, field_name, values, nulls); - } - } +void +extract_query_req(const google::protobuf::Message &msg, + const std::string &prefix, Datum *values, bool *nulls) +{ + using namespace google::protobuf; + + const Descriptor *descriptor = msg.GetDescriptor(); + const Reflection *reflection = msg.GetReflection(); + + for (int i = 0; i < descriptor->field_count(); ++i) + { + const FieldDescriptor *field = descriptor->field(i); + + // For now, we do not log any repeated fields plus they need special + // treatment. + if (field->is_repeated()) + { + continue; + } + + std::string curr_pref = prefix.empty() ? "" : prefix + "."; + std::string field_name = curr_pref + field->name().data(); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE && + field->message_type()->full_name() != "google.protobuf.Timestamp") + { + if (reflection->HasField(msg, field)) + { + const Message &nested = reflection->GetMessage(msg, field); + extract_query_req(nested, field_name, values, nulls); + } + } + else + { + process_field(field, reflection, msg, field_name, values, nulls); + } + } } diff --git a/gpcontrib/gp_stats_collector/src/log/LogSchema.h b/gpcontrib/gp_stats_collector/src/log/LogSchema.h index 8754741823a..f6c2247370a 100644 --- a/gpcontrib/gp_stats_collector/src/log/LogSchema.h +++ b/gpcontrib/gp_stats_collector/src/log/LogSchema.h @@ -25,7 +25,8 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef LOGSCHEMA_H +#define LOGSCHEMA_H #include #include @@ -37,26 +38,29 @@ extern "C" { #include "access/htup_details.h" #include "access/tupdesc.h" #include "catalog/pg_type.h" -#include "utils/timestamp.h" #include "utils/builtins.h" +#include "utils/timestamp.h" } -namespace google { -namespace protobuf { +namespace google +{ +namespace protobuf +{ class FieldDescriptor; class Message; class Reflection; class Timestamp; -} // namespace protobuf -} // namespace google +} // namespace protobuf +} // namespace google inline constexpr std::string_view schema_name = "gpsc"; inline constexpr std::string_view log_relname = "__log"; -struct LogDesc { - std::string_view pg_att_name; - std::string_view proto_field_name; - Oid type_oid; +struct LogDesc +{ + std::string_view pg_att_name; + std::string_view proto_field_name; + Oid type_oid; }; /* @@ -175,14 +179,14 @@ TupleDesc DescribeTuple(); Datum protots_to_timestamptz(const google::protobuf::Timestamp &ts); Datum field_to_datum(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg); + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg); /* Process a single proto field and store in values/nulls arrays */ void process_field(const google::protobuf::FieldDescriptor *field, - const google::protobuf::Reflection *reflection, - const google::protobuf::Message &msg, - const std::string &field_name, Datum *values, bool *nulls); + const google::protobuf::Reflection *reflection, + const google::protobuf::Message &msg, + const std::string &field_name, Datum *values, bool *nulls); /* * Extracts values from msg into values/nulls arrays. Caller must @@ -190,4 +194,6 @@ void process_field(const google::protobuf::FieldDescriptor *field, * to true for nested messages if parent message is missing). */ void extract_query_req(const google::protobuf::Message &msg, - const std::string &prefix, Datum *values, bool *nulls); + const std::string &prefix, Datum *values, bool *nulls); + +#endif /* LOGSCHEMA_H */ diff --git a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp index 4e3f6dae99f..de54a716016 100644 --- a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.cpp @@ -30,223 +30,287 @@ extern "C" { #include "postgres.h" -#include "utils/guc.h" +#include "access/htup.h" +#include "access/tupdesc.h" +#include "cdb/cdbexplain.h" #include "commands/dbcommands.h" -#include "commands/resgroupcmds.h" -#include "utils/builtins.h" -#include "utils/varlena.h" -#include "nodes/pg_list.h" #include "commands/explain.h" +#include "commands/resgroupcmds.h" #include "executor/instrument.h" -#include "access/tupdesc.h" -#include "access/htup.h" +#include "nodes/pg_list.h" +#include "stat_statements_parser/pg_stat_statements_parser.h" +#include "utils/builtins.h" #include "utils/elog.h" -#include "cdb/cdbexplain.h" -#include "stat_statements_parser/pg_stat_statements_ya_parser.h" +#include "utils/guc.h" +#include "utils/varlena.h" } -namespace { +namespace +{ template -auto wrap(Func &&func, Args &&...args) noexcept(!Throws) - -> decltype(func(std::forward(args)...)) { - - using RetType = decltype(func(std::forward(args)...)); - - // Empty struct for void return type. - struct VoidResult {}; - using ResultHolder = std::conditional_t, VoidResult, - std::optional>; - - bool success; - ErrorData *edata; - ResultHolder result_holder; - - PG_TRY(); - { - if constexpr (!std::is_void_v) { - result_holder.emplace(func(std::forward(args)...)); - } else { - func(std::forward(args)...); - } - edata = NULL; - success = true; - } - PG_CATCH(); - { - MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); - edata = CopyErrorData(); - MemoryContextSwitchTo(oldctx); - FlushErrorState(); - success = false; - } - PG_END_TRY(); - - if (!success) { - std::string err; - if (edata && edata->message) { - err = std::string(edata->message); - } else { - err = "Unknown error occurred"; - } - - if (edata) { - FreeErrorData(edata); - } - - if constexpr (Throws) { - throw std::runtime_error(err); - } - - if constexpr (!std::is_void_v) { - return RetType{}; - } else { - return; - } - } - - if constexpr (!std::is_void_v) { - return *std::move(result_holder); - } else { - return; - } +auto +wrap(Func &&func, Args &&...args) noexcept(!Throws) + -> decltype(func(std::forward(args)...)) +{ + using RetType = decltype(func(std::forward(args)...)); + + // Empty struct for void return type. + struct VoidResult + { + }; + using ResultHolder = std::conditional_t, VoidResult, + std::optional>; + + bool success; + ErrorData *edata; + ResultHolder result_holder; + + PG_TRY(); + { + if constexpr (!std::is_void_v) + { + result_holder.emplace(func(std::forward(args)...)); + } + else + { + func(std::forward(args)...); + } + edata = NULL; + success = true; + } + PG_CATCH(); + { + MemoryContext oldctx = MemoryContextSwitchTo(TopMemoryContext); + edata = CopyErrorData(); + MemoryContextSwitchTo(oldctx); + FlushErrorState(); + success = false; + } + PG_END_TRY(); + + if (!success) + { + std::string err; + if (edata && edata->message) + { + err = std::string(edata->message); + } + else + { + err = "Unknown error occurred"; + } + + if (edata) + { + FreeErrorData(edata); + } + + if constexpr (Throws) + { + throw std::runtime_error(err); + } + + if constexpr (!std::is_void_v) + { + return RetType{}; + } + else + { + return; + } + } + + if constexpr (!std::is_void_v) + { + return *std::move(result_holder); + } + else + { + return; + } } template -auto wrap_throw(Func &&func, Args &&...args) - -> decltype(func(std::forward(args)...)) { - return wrap(std::forward(func), std::forward(args)...); +auto +wrap_throw(Func &&func, Args &&...args) + -> decltype(func(std::forward(args)...)) +{ + return wrap(std::forward(func), std::forward(args)...); } template -auto wrap_noexcept(Func &&func, Args &&...args) noexcept - -> decltype(func(std::forward(args)...)) { - return wrap(std::forward(func), std::forward(args)...); +auto +wrap_noexcept(Func &&func, Args &&...args) noexcept + -> decltype(func(std::forward(args)...)) +{ + return wrap(std::forward(func), std::forward(args)...); } -} // namespace +} // namespace -void *gpdb::palloc(Size size) { return wrap_throw(::palloc, size); } +void * +gpdb::palloc(Size size) +{ + return wrap_throw(::palloc, size); +} -void *gpdb::palloc0(Size size) { return wrap_throw(::palloc0, size); } +void * +gpdb::palloc0(Size size) +{ + return wrap_throw(::palloc0, size); +} -char *gpdb::pstrdup(const char *str) { return wrap_throw(::pstrdup, str); } +char * +gpdb::pstrdup(const char *str) +{ + return wrap_throw(::pstrdup, str); +} -char *gpdb::get_database_name(Oid dbid) noexcept { - return wrap_noexcept(::get_database_name, dbid); +char * +gpdb::get_database_name(Oid dbid) noexcept +{ + return wrap_noexcept(::get_database_name, dbid); } -bool gpdb::split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept { - return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); +bool +gpdb::split_identifier_string(char *rawstring, char separator, + List **namelist) noexcept +{ + return wrap_noexcept(SplitIdentifierString, rawstring, separator, namelist); } -ExplainState gpdb::get_explain_state(QueryDesc *query_desc, - bool costs) noexcept { - return wrap_noexcept([&]() { - ExplainState *es = NewExplainState(); - es->costs = costs; - es->verbose = true; - es->format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(es); - ExplainPrintPlan(es, query_desc); - ExplainEndOutput(es); - return *es; - }); +ExplainState +gpdb::get_explain_state(QueryDesc *query_desc, bool costs) noexcept +{ + return wrap_noexcept([&]() { + ExplainState *es = NewExplainState(); + es->costs = costs; + es->verbose = true; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + ExplainPrintPlan(es, query_desc); + ExplainEndOutput(es); + return *es; + }); } -ExplainState gpdb::get_analyze_state(QueryDesc *query_desc, - bool analyze) noexcept { - return wrap_noexcept([&]() { - ExplainState *es = NewExplainState(); - es->analyze = analyze; - es->verbose = true; - es->buffers = es->analyze; - es->timing = es->analyze; - es->summary = es->analyze; - es->format = EXPLAIN_FORMAT_TEXT; - ExplainBeginOutput(es); - if (analyze) { - ExplainPrintPlan(es, query_desc); - ExplainPrintExecStatsEnd(es, query_desc); - } - ExplainEndOutput(es); - return *es; - }); +ExplainState +gpdb::get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept +{ + return wrap_noexcept([&]() { + ExplainState *es = NewExplainState(); + es->analyze = analyze; + es->verbose = true; + es->buffers = es->analyze; + es->timing = es->analyze; + es->summary = es->analyze; + es->format = EXPLAIN_FORMAT_TEXT; + ExplainBeginOutput(es); + if (analyze) + { + ExplainPrintPlan(es, query_desc); + ExplainPrintExecStatsEnd(es, query_desc); + } + ExplainEndOutput(es); + return *es; + }); } -Instrumentation *gpdb::instr_alloc(size_t n, int instrument_options, - bool async_mode) { - return wrap_throw(InstrAlloc, n, instrument_options, async_mode); +Instrumentation * +gpdb::instr_alloc(size_t n, int instrument_options, bool async_mode) +{ + return wrap_throw(InstrAlloc, n, instrument_options, async_mode); } -HeapTuple gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull) { - if (!tupleDescriptor || !values || !isnull) - throw std::runtime_error( - "Invalid input parameters for heap tuple formation"); +HeapTuple +gpdb::heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull) +{ + if (!tupleDescriptor || !values || !isnull) + throw std::runtime_error( + "Invalid input parameters for heap tuple formation"); - return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); + return wrap_throw(::heap_form_tuple, tupleDescriptor, values, isnull); } -void gpdb::pfree(void *pointer) noexcept { - // Note that ::pfree asserts that pointer != NULL. - if (!pointer) - return; +void +gpdb::pfree(void *pointer) noexcept +{ + // Note that ::pfree asserts that pointer != NULL. + if (!pointer) + return; - wrap_noexcept(::pfree, pointer); + wrap_noexcept(::pfree, pointer); } -MemoryContext gpdb::mem_ctx_switch_to(MemoryContext context) noexcept { - return MemoryContextSwitchTo(context); +MemoryContext +gpdb::mem_ctx_switch_to(MemoryContext context) noexcept +{ + return MemoryContextSwitchTo(context); } -const char *gpdb::get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept { - if (!name) - return nullptr; +const char * +gpdb::get_config_option(const char *name, bool missing_ok, + bool restrict_superuser) noexcept +{ + if (!name) + return nullptr; - return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); + return wrap_noexcept(GetConfigOption, name, missing_ok, restrict_superuser); } -void gpdb::list_free(List *list) noexcept { - if (!list) - return; +void +gpdb::list_free(List *list) noexcept +{ + if (!list) + return; - wrap_noexcept(::list_free, list); + wrap_noexcept(::list_free, list); } CdbExplain_ShowStatCtx * -gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime) { - if (!query_desc) - throw std::runtime_error("Invalid query descriptor"); +gpdb::cdbexplain_showExecStatsBegin(QueryDesc *query_desc, instr_time starttime) +{ + if (!query_desc) + throw std::runtime_error("Invalid query descriptor"); - return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); + return wrap_throw(::cdbexplain_showExecStatsBegin, query_desc, starttime); } -void gpdb::instr_end_loop(Instrumentation *instr) { - if (!instr) - throw std::runtime_error("Invalid instrumentation pointer"); +void +gpdb::instr_end_loop(Instrumentation *instr) +{ + if (!instr) + throw std::runtime_error("Invalid instrumentation pointer"); - wrap_throw(::InstrEndLoop, instr); + wrap_throw(::InstrEndLoop, instr); } -char *gpdb::gen_normquery(const char *query) noexcept { - return wrap_noexcept(::gen_normquery, query); +char * +gpdb::gen_normquery(const char *query) noexcept +{ + return wrap_noexcept(::gen_normquery, query); } -StringInfo gpdb::gen_normplan(const char *exec_plan) noexcept { - return wrap_noexcept(::gen_normplan, exec_plan); +StringInfo +gpdb::gen_normplan(const char *exec_plan) noexcept +{ + return wrap_noexcept(::gen_normplan, exec_plan); } -char *gpdb::get_rg_name_for_id(Oid group_id) { - return wrap_throw(GetResGroupNameForId, group_id); +char * +gpdb::get_rg_name_for_id(Oid group_id) +{ + return wrap_throw(GetResGroupNameForId, group_id); } -Oid gpdb::get_rg_id_by_session_id(int session_id) { - return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); +Oid +gpdb::get_rg_id_by_session_id(int session_id) +{ + return wrap_throw(ResGroupGetGroupIdBySessionId, session_id); } -void gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) { - return wrap_throw(::insert_log, req, utility); +void +gpdb::insert_log(const gpsc::SetQueryReq &req, bool utility) +{ + return wrap_throw(::insert_log, req, utility); } diff --git a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h index 576007f6c7c..5237b6be68a 100644 --- a/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h +++ b/gpcontrib/gp_stats_collector/src/memory/gpdbwrappers.h @@ -25,29 +25,32 @@ *------------------------------------------------------------------------- */ -#pragma once +#ifndef GPDBWRAPPERS_H +#define GPDBWRAPPERS_H extern "C" { #include "postgres.h" -#include "nodes/pg_list.h" +#include "access/htup.h" #include "commands/explain.h" #include "executor/instrument.h" -#include "access/htup.h" +#include "nodes/pg_list.h" #include "utils/elog.h" #include "utils/memutils.h" } -#include -#include #include -#include +#include #include +#include +#include -namespace gpsc { +namespace gpsc +{ class SetQueryReq; -} // namespace gpsc +} // namespace gpsc -namespace gpdb { +namespace gpdb +{ // Functions that call palloc(). // Make sure correct memory context is set. @@ -56,14 +59,14 @@ void *palloc0(Size size); char *pstrdup(const char *str); char *get_database_name(Oid dbid) noexcept; bool split_identifier_string(char *rawstring, char separator, - List **namelist) noexcept; + List **namelist) noexcept; ExplainState get_explain_state(QueryDesc *query_desc, bool costs) noexcept; ExplainState get_analyze_state(QueryDesc *query_desc, bool analyze) noexcept; Instrumentation *instr_alloc(size_t n, int instrument_options, bool async_mode); HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, - bool *isnull); + bool *isnull); CdbExplain_ShowStatCtx *cdbexplain_showExecStatsBegin(QueryDesc *query_desc, - instr_time starttime); + instr_time starttime); void instr_end_loop(Instrumentation *instr); char *gen_normquery(const char *query) noexcept; StringInfo gen_normplan(const char *executionPlan) noexcept; @@ -74,8 +77,10 @@ void insert_log(const gpsc::SetQueryReq &req, bool utility); void pfree(void *pointer) noexcept; MemoryContext mem_ctx_switch_to(MemoryContext context) noexcept; const char *get_config_option(const char *name, bool missing_ok, - bool restrict_superuser) noexcept; + bool restrict_superuser) noexcept; void list_free(List *list) noexcept; Oid get_rg_id_by_session_id(int session_id); -} // namespace gpdb +} // namespace gpdb + +#endif /* GPDBWRAPPERS_H */ diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c similarity index 82% rename from gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c index e24f53536a4..8e7bd917541 100644 --- a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c @@ -17,10 +17,10 @@ * specific language governing permissions and limitations * under the License. * - * pg_stat_statements_ya_parser.c + * pg_stat_statements_parser.c * * IDENTIFICATION - * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.c + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.c * *------------------------------------------------------------------------- */ @@ -42,7 +42,7 @@ #include "utils/memutils.h" #include "utils/queryjumble.h" -#include "pg_stat_statements_ya_parser.h" +#include "pg_stat_statements_parser.h" #ifndef FCONST #define FCONST 260 @@ -67,12 +67,14 @@ static bool need_replace(int token); static char *generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding); -void stat_statements_parser_init(void) +void +stat_statements_parser_init(void) { EnableQueryId(); } -void stat_statements_parser_deinit(void) +void +stat_statements_parser_deinit(void) { /* NO-OP */ } @@ -81,7 +83,8 @@ void stat_statements_parser_deinit(void) static bool need_replace(int token) { - return (token == FCONST) || (token == ICONST) || (token == SCONST) || (token == BCONST) || (token == XCONST); + return (token == FCONST) || (token == ICONST) || (token == SCONST) || + (token == BCONST) || (token == XCONST); } /* @@ -103,14 +106,11 @@ gen_normplan(const char *execution_plan) StringInfo plan_out = makeStringInfo(); ; - yyscanner = scanner_init(execution_plan, - &yyextra, + yyscanner = scanner_init(execution_plan, &yyextra, #if PG_VERSION_NUM >= 120000 - &ScanKeywords, - ScanKeywordTokens + &ScanKeywords, ScanKeywordTokens #else - ScanKeywords, - NumScanKeywords + ScanKeywords, NumScanKeywords #endif ); @@ -137,7 +137,8 @@ gen_normplan(const char *execution_plan) else { /* do not change - just copy as-is */ - tmp_str = strndup((char *)execution_plan + last_yylloc, yylloc - last_yylloc); + tmp_str = strndup((char *) execution_plan + last_yylloc, + yylloc - last_yylloc); appendStringInfoString(plan_out, tmp_str); free(tmp_str); } @@ -159,8 +160,8 @@ gen_normplan(const char *execution_plan) static int comp_location(const void *a, const void *b) { - int l = ((const LocationLen *) a)->location; - int r = ((const LocationLen *) b)->location; + int l = ((const LocationLen *) a)->location; + int r = ((const LocationLen *) b)->location; if (l < r) return -1; @@ -199,35 +200,32 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) core_yyscan_t yyscanner; core_yy_extra_type yyextra; core_YYSTYPE yylval; - YYLTYPE yylloc; - int last_loc = -1; - int i; + YYLTYPE yylloc; + int last_loc = -1; + int i; /* * Sort the records by location so that we can process them in order while * scanning the query text. */ if (jstate->clocations_count > 1) - qsort(jstate->clocations, jstate->clocations_count, - sizeof(LocationLen), comp_location); + qsort(jstate->clocations, jstate->clocations_count, sizeof(LocationLen), + comp_location); locs = jstate->clocations; /* initialize the flex scanner --- should match raw_parser() */ - yyscanner = scanner_init(query, - &yyextra, - &ScanKeywords, - ScanKeywordTokens); + yyscanner = scanner_init(query, &yyextra, &ScanKeywords, ScanKeywordTokens); /* Search for each constant, in sequence */ for (i = 0; i < jstate->clocations_count; i++) { - int loc = locs[i].location; - int tok; + int loc = locs[i].location; + int tok; Assert(loc >= 0); if (loc <= last_loc) - continue; /* Duplicate constant, ignore */ + continue; /* Duplicate constant, ignore */ /* Lex tokens until we find the desired constant */ for (;;) @@ -236,7 +234,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) /* We should not hit end-of-string, but if we do, behave sanely */ if (tok == 0) - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ /* * We should find the token position exactly, but if we somehow @@ -260,7 +258,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) */ tok = core_yylex(&yylval, &yylloc, yyscanner); if (tok == 0) - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ } /* @@ -268,7 +266,7 @@ fill_in_constant_lengths(JumbleState *jstate, const char *query) * byte after the text of the current token in scanbuf. */ locs[i].length = strlen(yyextra.scanbuf + loc); - break; /* out of inner for-loop */ + break; /* out of inner for-loop */ } } @@ -299,14 +297,13 @@ static char * generate_normalized_query(JumbleState *jstate, const char *query, int *query_len_p, int encoding) { - char *norm_query; - int query_len = *query_len_p; - int i, - len_to_wrt, /* Length (in bytes) to write */ - quer_loc = 0, /* Source query byte location */ - n_quer_loc = 0, /* Normalized query byte location */ - last_off = 0, /* Offset from start for previous tok */ - last_tok_len = 0; /* Length (in bytes) of that tok */ + char *norm_query; + int query_len = *query_len_p; + int i, len_to_wrt, /* Length (in bytes) to write */ + quer_loc = 0, /* Source query byte location */ + n_quer_loc = 0, /* Normalized query byte location */ + last_off = 0, /* Offset from start for previous tok */ + last_tok_len = 0; /* Length (in bytes) of that tok */ /* * Get constants' lengths (core system only gives us locations). Note @@ -319,14 +316,14 @@ generate_normalized_query(JumbleState *jstate, const char *query, for (i = 0; i < jstate->clocations_count; i++) { - int off, /* Offset from start for cur tok */ - tok_len; /* Length (in bytes) of that tok */ + int off, /* Offset from start for cur tok */ + tok_len; /* Length (in bytes) of that tok */ off = jstate->clocations[i].location; tok_len = jstate->clocations[i].length; if (tok_len < 0) - continue; /* ignore any duplicates */ + continue; /* ignore any duplicates */ /* Copy next chunk (what precedes the next constant) */ len_to_wrt = off - last_off; @@ -361,18 +358,21 @@ generate_normalized_query(JumbleState *jstate, const char *query, return norm_query; } -char *gen_normquery(const char *query) +char * +gen_normquery(const char *query) { - if (!query) { + if (!query) + { return NULL; } JumbleState jstate; - jstate.jumble = (unsigned char *)palloc(JUMBLE_SIZE); + jstate.jumble = (unsigned char *) palloc(JUMBLE_SIZE); jstate.jumble_len = 0; jstate.clocations_buf_size = 32; - jstate.clocations = (LocationLen *) - palloc(jstate.clocations_buf_size * sizeof(LocationLen)); + jstate.clocations = (LocationLen *) palloc(jstate.clocations_buf_size * + sizeof(LocationLen)); jstate.clocations_count = 0; int query_len = strlen(query); - return generate_normalized_query(&jstate, query, &query_len, GetDatabaseEncoding()); + return generate_normalized_query(&jstate, query, &query_len, + GetDatabaseEncoding()); } \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h similarity index 87% rename from gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h rename to gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h index a613ba04259..b6c5dea7b36 100644 --- a/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h +++ b/gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h @@ -17,19 +17,19 @@ * specific language governing permissions and limitations * under the License. * - * pg_stat_statements_ya_parser.h + * pg_stat_statements_parser.h * * IDENTIFICATION - * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_ya_parser.h + * gpcontrib/gp_stats_collector/src/stat_statements_parser/pg_stat_statements_parser.h * *------------------------------------------------------------------------- */ -#pragma once +#ifndef PG_STAT_STATEMENTS_PARSER_H +#define PG_STAT_STATEMENTS_PARSER_H #ifdef __cplusplus -extern "C" -{ +extern "C" { #endif extern void stat_statements_parser_init(void); @@ -41,3 +41,5 @@ char *gen_normquery(const char *query); #ifdef __cplusplus } #endif + +#endif /* PG_STAT_STATEMENTS_PARSER_H */ diff --git a/pom.xml b/pom.xml index dbc67b99a5f..1faa566fcec 100644 --- a/pom.xml +++ b/pom.xml @@ -1275,9 +1275,6 @@ code or new licensing patterns. introduced by Cloudberry. --> gpcontrib/gp_stats_collector/gp_stats_collector.control - gpcontrib/gp_stats_collector/protos/gpsc_set_service.proto - gpcontrib/gp_stats_collector/protos/gpsc_plan.proto - gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto gpcontrib/gp_stats_collector/.clang-format gpcontrib/gp_stats_collector/Makefile From 3b8ec62352db36a876e5e9cdfc83630bca80279b Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 14:15:58 +0300 Subject: [PATCH 105/167] [gp_stats_collector] Wrap hook call in try/catch on error path Add PG_TRY/PG_CATCH around query_info_collect_hook in PortalCleanup error path to prevent exceptions from propagating during cleanup. --- src/backend/commands/portalcmds.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/portalcmds.c b/src/backend/commands/portalcmds.c index 0ea5874e884..e23dc1d9c43 100644 --- a/src/backend/commands/portalcmds.c +++ b/src/backend/commands/portalcmds.c @@ -374,10 +374,22 @@ PortalCleanup(Portal portal) FreeQueryDesc(queryDesc); CurrentResourceOwner = saveResourceOwner; - } else { + } + else + { /* GPDB hook for collecting query info */ if (queryDesc->gpsc_query_key && query_info_collect_hook) - (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); + { + PG_TRY(); + { + (*query_info_collect_hook)(METRICS_QUERY_ERROR, queryDesc); + } + PG_CATCH(); + { + FlushErrorState(); + } + PG_END_TRY(); + } } } From 15dff28eddcc98e40243a42d46f1efc7a9a99328 Mon Sep 17 00:00:00 2001 From: NJrslv Date: Tue, 31 Mar 2026 14:33:20 +0300 Subject: [PATCH 106/167] [gp_stats_collector] Adapt namings for Cloudberry Rename ON MASTER to ON COORDINATOR in test SQL. Prefer pg_usleep() over std::this_thread::sleep_for(). Add pg_unreachable() after ereport(ERROR). Widen motion stats fields to uint64. --- .../gp_stats_collector--1.0--1.1.sql | 10 +++++----- .../gp_stats_collector--1.0.sql | 6 +++--- .../gp_stats_collector--1.1.sql | 16 ++++++++-------- .../gp_stats_collector/protos/gpsc_metrics.proto | 6 +++--- .../gp_stats_collector/src/UDSConnector.cpp | 4 +--- .../gp_stats_collector/src/hook_wrappers.cpp | 1 + 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql index 4e0157117e9..398f03b4fa9 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0--1.1.sql @@ -25,7 +25,7 @@ DROP FUNCTION __gpsc_stat_messages_reset_f_on_master(); CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -39,12 +39,12 @@ $$ SELECT gpsc.__stat_messages_reset_f_on_master(); SELECT gpsc.__stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_segments() RETURNS SETOF record @@ -77,7 +77,7 @@ ORDER BY segid; CREATE FUNCTION gpsc.__init_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__init_log_on_segments() RETURNS SETOF void @@ -97,7 +97,7 @@ CREATE VIEW gpsc.log AS CREATE FUNCTION gpsc.__truncate_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__truncate_log_on_segments() RETURNS SETOF void diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql index ec902b02e02..e4a50aa2133 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.0.sql @@ -6,7 +6,7 @@ CREATE FUNCTION __gpsc_stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -20,12 +20,12 @@ $$ SELECT __gpsc_stat_messages_reset_f_on_master(); SELECT __gpsc_stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION __gpsc_stat_messages_f_on_segments() RETURNS SETOF record diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql index 6e24207e913..3ebdad14b06 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1.sql @@ -8,7 +8,7 @@ CREATE SCHEMA gpsc; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() RETURNS SETOF void @@ -22,12 +22,12 @@ $$ SELECT gpsc.__stat_messages_reset_f_on_master(); SELECT gpsc.__stat_messages_reset_f_on_segments(); $$ -LANGUAGE SQL EXECUTE ON MASTER; +LANGUAGE SQL EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_master() RETURNS SETOF record AS 'MODULE_PATHNAME', 'gpsc_stat_messages' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__stat_messages_f_on_segments() RETURNS SETOF record @@ -59,7 +59,7 @@ ORDER BY segid; CREATE FUNCTION gpsc.__init_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_init_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__init_log_on_segments() RETURNS SETOF void @@ -79,7 +79,7 @@ ORDER BY tmid, ssid, ccnt; CREATE FUNCTION gpsc.__truncate_log_on_master() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_truncate_log' -LANGUAGE C STRICT VOLATILE EXECUTE ON MASTER; +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__truncate_log_on_segments() RETURNS SETOF void @@ -97,14 +97,14 @@ $$ LANGUAGE plpgsql VOLATILE; CREATE FUNCTION gpsc.__test_uds_start_server(path text) RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' -LANGUAGE C STRICT EXECUTE ON MASTER; +LANGUAGE C STRICT EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) RETURNS SETOF bigint AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' -LANGUAGE C STRICT EXECUTE ON MASTER; +LANGUAGE C STRICT EXECUTE ON COORDINATOR; CREATE FUNCTION gpsc.__test_uds_stop_server() RETURNS SETOF void AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' -LANGUAGE C EXECUTE ON MASTER; +LANGUAGE C EXECUTE ON COORDINATOR; diff --git a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto index 7853dc58db7..10991301557 100644 --- a/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto +++ b/gpcontrib/gp_stats_collector/protos/gpsc_metrics.proto @@ -113,9 +113,9 @@ message SystemStat { } message NetworkStat { - uint32 total_bytes = 1; - uint32 tuple_bytes = 2; - uint32 chunks = 3; + uint64 total_bytes = 1; + uint64 tuple_bytes = 2; + uint64 chunks = 3; } message InterconnectStat { diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp index 16344366456..056fa9071a5 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -31,13 +31,11 @@ #include "log/LogOps.h" #include "memory/gpdbwrappers.h" -#include #include #include #include #include #include -#include #include extern "C" { @@ -132,7 +130,7 @@ UDSConnector::report_query(const gpsc::SetQueryReq &req, // if a message does not fit a single packet, we take a nap // before sending the next one. // Otherwise, MSG_DONTWAIT send might overflow the UDS - (std::this_thread::sleep_for(std::chrono::milliseconds(1)), true)); + (pg_usleep(1000), true)); if (sent < 0) { diff --git a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp index 3f19d4d9930..38ea117bda2 100644 --- a/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp +++ b/gpcontrib/gp_stats_collector/src/hook_wrappers.cpp @@ -116,6 +116,7 @@ cpp_call(T *obj, R (T::*func)(Args...), Args... args) catch (const std::exception &e) { ereport(ERROR, (errmsg("Unexpected exception in gpsc %s", e.what()))); + pg_unreachable(); } } From 7d65d3ae81543fa2caafe9971725065c5747368c Mon Sep 17 00:00:00 2001 From: NJrslv Date: Wed, 1 Apr 2026 10:39:45 +0300 Subject: [PATCH 107/167] [gp_stats_collector] Remove unnecessary CONFIGURE_EXTRA_OPTS param --- .../automation/cloudberry/scripts/configure-cloudberry.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index a9086a434fb..90f0614bfe8 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -53,7 +53,6 @@ # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) -# CONFIGURE_EXTRA_OPTS - Args to pass to configure command # ENABLE_DEBUG - Enable debug build options (true/false, defaults to # false) # @@ -179,8 +178,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-uuid=e2fs \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib \ - ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" # Capture version information From 66c8f00e148db2f5098f9f12d1c7424f68ccb7d7 Mon Sep 17 00:00:00 2001 From: zhangyue Date: Fri, 3 Apr 2026 12:36:41 +0800 Subject: [PATCH 108/167] Fix sed -i compatibility in configure.ac Apply the same macOS BSD sed fix from 7e867f6 to configure.ac, so that users regenerating configure from source also get the compatibility fix. --- configure.ac | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 3ca5585f1b6..ac174a47824 100644 --- a/configure.ac +++ b/configure.ac @@ -3238,4 +3238,11 @@ AC_OUTPUT # The configure args contain '-Wl,-rpath,\$$ORIGIN`, when it falls # as a C literal string, it's invalid, so converting `\` to `\\` # to be correct for C program. -sed -i '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h +case $build_os in +darwin*) + sed -i '' '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h + ;; +*) + sed -i '/define CONFIGURE_ARGS/s,\([[^\\]]\)\\\$\$,\1\\\\$$,g' src/include/pg_config.h + ;; +esac From 7306eb3105574c66db93a59f8fe3f9b20337299d Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 31 Mar 2026 17:56:30 +0800 Subject: [PATCH 109/167] Disable mdblocales by default in configure Changed the default value of `--with-mdblocales` from 'yes' to 'no' in configure.ac. This prevents enabling custom locales by default, which require additional packages and may cause compatibility issues across different system environments (e.g., Rocky Linux 8 vs 9 with different libc versions). Users who need mdblocales can explicitly enable it with `--with-mdblocales` option. Regenerated configure file using autoconf 2.69 on Rocky Linux 8. --- configure | 104 ++++++++++++++++++++++++++------------------------- configure.ac | 2 +- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/configure b/configure index bcbe741a543..203ddc20cb0 100755 --- a/configure +++ b/configure @@ -1700,7 +1700,7 @@ Optional Packages: --without-libcurl do not use libcurl --with-apr-config=PATH path to apr-1-config utility --with-gnu-ld assume the C compiler uses GNU ld [default=no] - --without-mdblocales build without MDB locales + --with-mdblocales build with MDB locales --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -2916,6 +2916,8 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu PG_PACKAGE_VERSION=14.7 + + ac_aux_dir= for ac_dir in config "$srcdir"/config; do if test -f "$ac_dir/install-sh"; then @@ -13022,56 +13024,6 @@ $as_echo "${python_libspec} ${python_additional_libs}" >&6; } -fi - -if test "$with_mdblocales" = yes; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 -$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } -if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lmdblocales $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char mdb_setlocale (); -int -main () -{ -return mdb_setlocale (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_mdblocales_mdb_setlocale=yes -else - ac_cv_lib_mdblocales_mdb_setlocale=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 -$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } -if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBMDBLOCALES 1 -_ACEOF - - LIBS="-lmdblocales $LIBS" - -else - as_fn_error $? "mdblocales library not found" "$LINENO" 5 -fi - fi if test x"$cross_compiling" = x"yes" && test -z "$with_system_tzdata"; then @@ -14996,6 +14948,56 @@ fi fi +if test "$with_mdblocales" = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for mdb_setlocale in -lmdblocales" >&5 +$as_echo_n "checking for mdb_setlocale in -lmdblocales... " >&6; } +if ${ac_cv_lib_mdblocales_mdb_setlocale+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lmdblocales $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char mdb_setlocale (); +int +main () +{ +return mdb_setlocale (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_mdblocales_mdb_setlocale=yes +else + ac_cv_lib_mdblocales_mdb_setlocale=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mdblocales_mdb_setlocale" >&5 +$as_echo "$ac_cv_lib_mdblocales_mdb_setlocale" >&6; } +if test "x$ac_cv_lib_mdblocales_mdb_setlocale" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBMDBLOCALES 1 +_ACEOF + + LIBS="-lmdblocales $LIBS" + +else + as_fn_error $? "mdblocales library not found" "$LINENO" 5 +fi + +fi + if test "$enable_external_fts" = yes; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for jansson_version_str in -ljansson" >&5 $as_echo_n "checking for jansson_version_str in -ljansson... " >&6; } diff --git a/configure.ac b/configure.ac index ac174a47824..a51eb685585 100644 --- a/configure.ac +++ b/configure.ac @@ -1508,7 +1508,7 @@ AC_SUBST(install_bin) # MDB locales # -PGAC_ARG_BOOL(with, mdblocales, yes, [build without MDB locales], +PGAC_ARG_BOOL(with, mdblocales, no, [build with MDB locales], [AC_DEFINE([USE_MDBLOCALES], 1, [Define to 1 to build with MDB locales. (--with-mdblocales)])]) AC_SUBST(USE_MDBLOCALES) From 707ff3b28a25c522681b3f1dfc63e1fa9ea95dc4 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 31 Mar 2026 17:56:30 +0800 Subject: [PATCH 110/167] Regenerate configure for gp_stats_clolector Regenerated configure file using autoconf 2.69 on Rocky Linux 8. --- configure | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/configure b/configure index 203ddc20cb0..3cac3c4eaed 100755 --- a/configure +++ b/configure @@ -723,7 +723,9 @@ with_libcurl with_rt with_zstd with_yezzey +PROTOC with_gp_stats_collector +with_zstd with_libbz2 LZ4_LIBS LZ4_CFLAGS @@ -1696,6 +1698,8 @@ Optional Packages: --with-lz4 build with LZ4 support --without-libbz2 do not use bzip2 --without-zstd do not build with Zstandard + --with-gp_stats_collector + build with stats collector extension --without-rt do not use Realtime Library --without-libcurl do not use libcurl --with-apr-config=PATH path to apr-1-config utility @@ -11151,6 +11155,155 @@ fi $as_echo "$with_zstd" >&6; } +# +# gp_stats_collector +# + + + +# Check whether --with-gp_stats_collector was given. +if test "${with_gp_stats_collector+set}" = set; then : + withval=$with_gp_stats_collector; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-gp_stats_collector option" "$LINENO" 5 + ;; + esac + +else + with_gp_stats_collector=no + +fi + + + + +if test "$with_gp_stats_collector" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for protobuf >= 3.0.0" >&5 +$as_echo_n "checking for protobuf >= 3.0.0... " >&6; } + +if test -n "$PROTOBUF_CFLAGS"; then + pkg_cv_PROTOBUF_CFLAGS="$PROTOBUF_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"protobuf >= 3.0.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "protobuf >= 3.0.0") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PROTOBUF_CFLAGS=`$PKG_CONFIG --cflags "protobuf >= 3.0.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PROTOBUF_LIBS"; then + pkg_cv_PROTOBUF_LIBS="$PROTOBUF_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"protobuf >= 3.0.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "protobuf >= 3.0.0") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PROTOBUF_LIBS=`$PKG_CONFIG --libs "protobuf >= 3.0.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PROTOBUF_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "protobuf >= 3.0.0" 2>&1` + else + PROTOBUF_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "protobuf >= 3.0.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PROTOBUF_PKG_ERRORS" >&5 + + as_fn_error $? "protobuf >= 3.0.0 is required for gp_stats_collector" "$LINENO" 5 + +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "protobuf >= 3.0.0 is required for gp_stats_collector" "$LINENO" 5 + +else + PROTOBUF_CFLAGS=$pkg_cv_PROTOBUF_CFLAGS + PROTOBUF_LIBS=$pkg_cv_PROTOBUF_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + # Extract the first word of "protoc", so it can be a program name with args. +set dummy protoc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PROTOC+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PROTOC in + [\\/]* | ?:[\\/]*) + ac_cv_path_PROTOC="$PROTOC" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PROTOC="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_path_PROTOC" && ac_cv_path_PROTOC="no" + ;; +esac +fi +PROTOC=$ac_cv_path_PROTOC +if test -n "$PROTOC"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROTOC" >&5 +$as_echo "$PROTOC" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test "$PROTOC" = no; then + as_fn_error $? "protoc is required for gp_stats_collector but was not found in PATH" "$LINENO" 5 + fi +fi + if test "$with_zstd" = yes; then pkg_failed=no From 86f110732cb774d5801310442254e7ca9b1eafa8 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Thu, 2 Apr 2026 19:11:43 +0800 Subject: [PATCH 111/167] Add missing runtime dependencies for minimal installations Add essential command-line utilities as runtime dependencies to ensure proper operation in minimal system installations such as container environments. Added dependencies: - which/debianutils: Required by gp_bash_functions.sh to locate binaries - hostname: Used by various scripts to identify the system - less: Required for paging output in interactive sessions These utilities may not be present in minimal installations (e.g., Oracle Linux 9 containers, Ubuntu minimal images), causing runtime failures during database initialization and operation. Changes: - RPM spec: Add hostname, less, and which to Requires - DEB control (Ubuntu 22.04/24.04): Add hostname and debianutils to Depends (less already present) --- devops/build/packaging/deb/ubuntu22.04/control | 2 ++ devops/build/packaging/deb/ubuntu24.04/control | 2 ++ .../build/packaging/rpm/apache-cloudberry-db-incubating.spec | 3 +++ 3 files changed, 7 insertions(+) diff --git a/devops/build/packaging/deb/ubuntu22.04/control b/devops/build/packaging/deb/ubuntu22.04/control index 4bc5d90b84d..6b05863b780 100644 --- a/devops/build/packaging/deb/ubuntu22.04/control +++ b/devops/build/packaging/deb/ubuntu22.04/control @@ -46,6 +46,8 @@ Provides: apache-cloudberry-db Architecture: any Depends: curl, cgroup-tools, + debianutils, + hostname, iputils-ping, iproute2, keyutils, diff --git a/devops/build/packaging/deb/ubuntu24.04/control b/devops/build/packaging/deb/ubuntu24.04/control index a561d8a4386..9e2c3eab451 100644 --- a/devops/build/packaging/deb/ubuntu24.04/control +++ b/devops/build/packaging/deb/ubuntu24.04/control @@ -46,6 +46,8 @@ Provides: apache-cloudberry-db Architecture: amd64 Depends: curl, cgroup-tools, + debianutils, + hostname, iputils-ping, iproute2, keyutils, diff --git a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec index 03fa0a34570..517b35212bf 100644 --- a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec +++ b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec @@ -52,12 +52,15 @@ Prefix: %{cloudberry_install_dir} # List runtime dependencies Requires: bash +Requires: hostname Requires: iproute Requires: iputils +Requires: less Requires: openssh Requires: openssh-clients Requires: openssh-server Requires: rsync +Requires: which %if 0%{?rhel} == 8 Requires: apr From 86d572369b7edfa1d03e979ab73611922600296a Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Fri, 3 Apr 2026 21:17:59 +0800 Subject: [PATCH 112/167] Fix compliance issues for RPM and DEB packages As an Apache incubating project, convenience binaries must include LICENSE, NOTICE, and DISCLAIMER files. This commit adds these mandatory compliance files into the spec and rules definitions to ensure they are properly distributed with the binary RPM and DEB packages. Additionally, this commit: - Automates the cp of the .spec file into the ~/rpmbuild tree to prevent build failures for new users. - Dynamically locates debian metadata from OS-specific directories and copies to project root for dpkg-buildpackage. - Generates debian/copyright file by combining LICENSE and NOTICE to meet Debian policy requirements. --- devops/build/packaging/deb/build-deb.sh | 47 +++++++++++++++---- devops/build/packaging/deb/ubuntu22.04/rules | 17 ++++++- .../rpm/apache-cloudberry-db-incubating.spec | 8 +++- devops/build/packaging/rpm/build-rpm.sh | 40 +++++++++++++++- 4 files changed, 99 insertions(+), 13 deletions(-) diff --git a/devops/build/packaging/deb/build-deb.sh b/devops/build/packaging/deb/build-deb.sh index 1f5aef2258a..61a29e50fc9 100755 --- a/devops/build/packaging/deb/build-deb.sh +++ b/devops/build/packaging/deb/build-deb.sh @@ -109,7 +109,7 @@ export CBDB_FULL_VERSION=$VERSION # Set version if not provided if [ -z "${VERSION}" ]; then - export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) + export CBDB_FULL_VERSION=$(./getversion 2>/dev/null | cut -d'-' -f 1 | cut -d'+' -f 1 || echo "unknown") fi if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then @@ -127,22 +127,48 @@ fi # Detect OS distribution (e.g., ubuntu22.04, debian12) if [ -z ${OS_DISTRO+x} ]; then if [ -f /etc/os-release ]; then + # Temporarily disable unbound variable check for sourcing os-release + set +u . /etc/os-release - OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') + set -u + # Ensure ID and VERSION_ID are set before using them + OS_DISTRO=$(echo "${ID:-unknown}${VERSION_ID:-}" | tr '[:upper:]' '[:lower:]') else OS_DISTRO="unknown" fi fi +# Ensure OS_DISTRO is exported and not empty +export OS_DISTRO=${OS_DISTRO:-unknown} + export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} # Check if required commands are available check_commands -# Define the control file path -CONTROL_FILE=debian/control +# Find project root (assumed to be four levels up from scripts directory: devops/build/packaging/deb/) +PROJECT_ROOT="$(cd "$(dirname "$0")/../../../../" && pwd)" + +# Define where the debian metadata is located +DEBIAN_SRC_DIR="$(dirname "$0")/${OS_DISTRO}" + +# Prepare the debian directory at the project root (required by dpkg-buildpackage) +if [ -d "$DEBIAN_SRC_DIR" ]; then + echo "Preparing debian directory from $DEBIAN_SRC_DIR..." + mkdir -p "$PROJECT_ROOT/debian" + # Use /. to copy directory contents if target exists instead of nested directories + cp -rf "$DEBIAN_SRC_DIR"/. "$PROJECT_ROOT/debian/" +else + if [ ! -d "$PROJECT_ROOT/debian" ]; then + echo "Error: Debian metadata not found at $DEBIAN_SRC_DIR and no debian/ directory exists at root." + exit 1 + fi +fi + +# Define the control file path (at the project root) +CONTROL_FILE="$PROJECT_ROOT/debian/control" -# Check if the spec file exists +# Check if the control file exists if [ ! -f "$CONTROL_FILE" ]; then echo "Error: Control file not found at $CONTROL_FILE." exit 1 @@ -160,10 +186,15 @@ if [ "${DRY_RUN:-false}" = true ]; then exit 0 fi -# Run debbuild with the provided options -echo "Building DEB with Version $CBDB_FULL_VERSION ..." +# Run debbuild from the project root +echo "Building DEB with Version $CBDB_FULL_VERSION in $PROJECT_ROOT ..." + +print_changelog > "$PROJECT_ROOT/debian/changelog" -print_changelog > debian/changelog +# Only cd if we are not already at the project root +if [ "$(pwd)" != "$PROJECT_ROOT" ]; then + cd "$PROJECT_ROOT" +fi if ! eval "$DEBBUILD_CMD"; then echo "Error: deb build failed." diff --git a/devops/build/packaging/deb/ubuntu22.04/rules b/devops/build/packaging/deb/ubuntu22.04/rules index cb387d209e6..463486cf03f 100755 --- a/devops/build/packaging/deb/ubuntu22.04/rules +++ b/devops/build/packaging/deb/ubuntu22.04/rules @@ -19,7 +19,22 @@ include /usr/share/dpkg/default.mk dh $@ --parallel gpinstall: - make install DESTDIR=${DEBIAN_DESTINATION} prefix= + # If the build staging directory is empty, copy from the pre-installed location. + # In CI, BUILD_DESTINATION already points here so it will be populated. + # For local manual packaging, copy from the installed Cloudberry path. + @mkdir -p ${DEBIAN_DESTINATION} + @if [ -z "$$(ls -A ${DEBIAN_DESTINATION} 2>/dev/null)" ]; then \ + echo "Copying pre-built binaries from ${CBDB_BIN_PATH} to ${DEBIAN_DESTINATION}..."; \ + cp -a ${CBDB_BIN_PATH}/* ${DEBIAN_DESTINATION}/; \ + else \ + echo "Build staging directory already populated, skipping copy."; \ + fi + # Copy Apache compliance files into the build staging directory + cp -a LICENSE NOTICE DISCLAIMER ${DEBIAN_DESTINATION}/ + cp -a licenses ${DEBIAN_DESTINATION}/ + # Create debian/copyright for Debian policy compliance + mkdir -p $(shell pwd)/debian + cat LICENSE NOTICE > $(shell pwd)/debian/copyright override_dh_auto_install: gpinstall # the staging directory for creating a debian is NOT the right GPHOME. diff --git a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec index 517b35212bf..e228f8fe76a 100644 --- a/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec +++ b/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec @@ -155,6 +155,12 @@ mkdir -p %{buildroot}%{cloudberry_install_dir}-%{version} cp -R %{cloudberry_install_dir}/* %{buildroot}%{cloudberry_install_dir}-%{version} +# Copy Apache mandatory compliance files from the SOURCES directory into the installation directory +cp %{_sourcedir}/LICENSE %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp %{_sourcedir}/NOTICE %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp %{_sourcedir}/DISCLAIMER %{buildroot}%{cloudberry_install_dir}-%{version}/ +cp -R %{_sourcedir}/licenses %{buildroot}%{cloudberry_install_dir}-%{version}/ + # Create the symbolic link ln -sfn %{cloudberry_install_dir}-%{version} %{buildroot}%{cloudberry_install_dir} @@ -162,8 +168,6 @@ ln -sfn %{cloudberry_install_dir}-%{version} %{buildroot}%{cloudberry_install_di %{prefix}-%{version} %{prefix} -%license %{cloudberry_install_dir}-%{version}/LICENSE - %debug_package %post diff --git a/devops/build/packaging/rpm/build-rpm.sh b/devops/build/packaging/rpm/build-rpm.sh index ceb7d18d392..2c490166f45 100755 --- a/devops/build/packaging/rpm/build-rpm.sh +++ b/devops/build/packaging/rpm/build-rpm.sh @@ -118,10 +118,46 @@ fi # Check if required commands are available check_commands -# Define the spec file path +# Define the source spec file path (assuming it is in the same directory as the script) +SOURCE_SPEC_FILE="$(dirname "$0")/apache-cloudberry-db-incubating.spec" + +# Ensure rpmbuild SPECS and SOURCES directories exist +mkdir -p ~/rpmbuild/SPECS +mkdir -p ~/rpmbuild/SOURCES + +# Find project root (assumed to be four levels up from scripts directory: devops/build/packaging/rpm/) +PROJECT_ROOT="$(cd "$(dirname "$0")/../../../../" && pwd)" + +# Define the target spec file path SPEC_FILE=~/rpmbuild/SPECS/apache-cloudberry-db-incubating.spec -# Check if the spec file exists +# Copy the spec file to rpmbuild/SPECS if the source exists and is different +if [ -f "$SOURCE_SPEC_FILE" ]; then + # Avoid copying if SPEC_FILE is already a symlink/file pointing to SOURCE_SPEC_FILE (common in CI) + if [ ! "$SOURCE_SPEC_FILE" -ef "$SPEC_FILE" ]; then + cp -f "$SOURCE_SPEC_FILE" "$SPEC_FILE" + fi +else + echo "Warning: Source spec file not found at $SOURCE_SPEC_FILE, assuming it is already in ~/rpmbuild/SPECS/" +fi + +# Copy Apache mandatory compliance files to rpmbuild/SOURCES +echo "Copying compliance files from $PROJECT_ROOT to ~/rpmbuild/SOURCES..." +for f in LICENSE NOTICE DISCLAIMER; do + if [ -f "$PROJECT_ROOT/$f" ]; then + cp -af "$PROJECT_ROOT/$f" ~/rpmbuild/SOURCES/ + else + echo "Warning: $f not found in $PROJECT_ROOT" + fi +done + +if [ -d "$PROJECT_ROOT/licenses" ]; then + cp -af "$PROJECT_ROOT/licenses" ~/rpmbuild/SOURCES/ +else + echo "Warning: licenses directory not found in $PROJECT_ROOT" +fi + +# Check if the spec file exists at the target location before proceeding if [ ! -f "$SPEC_FILE" ]; then echo "Error: Spec file not found at $SPEC_FILE." exit 1 From 843e15b2d2e890cdc62624f394ac2477d6bd3cf2 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 7 Apr 2026 11:40:07 +0800 Subject: [PATCH 113/167] Build: make diskquota installation opt-in diskquota is currently built and installed by default from gpcontrib, which makes it available in all standard builds without an explicit decision to enable it. Change diskquota to an opt-in installation model by adding a dedicated `--with-diskquota` configure option that defaults to `no`. Only build, install, and run installcheck for diskquota when that option is explicitly enabled. This makes diskquota adoption an explicit choice and avoids shipping the extension by default. --- configure | 31 +++++++++++++++++++ configure.ac | 7 +++++ .../scripts/configure-cloudberry.sh | 1 + gpcontrib/Makefile | 12 ++++--- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 3cac3c4eaed..586e4d0cf6a 100755 --- a/configure +++ b/configure @@ -725,6 +725,7 @@ with_zstd with_yezzey PROTOC with_gp_stats_collector +with_diskquota with_zstd with_libbz2 LZ4_LIBS @@ -1698,6 +1699,7 @@ Optional Packages: --with-lz4 build with LZ4 support --without-libbz2 do not use bzip2 --without-zstd do not build with Zstandard + --with-diskquota build with diskquota extension --with-gp_stats_collector build with stats collector extension --without-rt do not use Realtime Library @@ -11155,6 +11157,35 @@ fi $as_echo "$with_zstd" >&6; } +# +# diskquota +# + + + +# Check whether --with-diskquota was given. +if test "${with_diskquota+set}" = set; then : + withval=$with_diskquota; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-diskquota option" "$LINENO" 5 + ;; + esac + +else + with_diskquota=no + +fi + + + + # # gp_stats_collector # diff --git a/configure.ac b/configure.ac index a51eb685585..58fa651ee2e 100644 --- a/configure.ac +++ b/configure.ac @@ -1368,6 +1368,13 @@ PGAC_ARG_BOOL(with, zstd, yes, [do not build with Zstandard], AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) +# +# diskquota +# +PGAC_ARG_BOOL(with, diskquota, no, + [build with diskquota extension]) +AC_SUBST(with_diskquota) + # # gp_stats_collector # diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 90f0614bfe8..80575309092 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -162,6 +162,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ + --with-diskquota \ --with-gp-stats-collector \ --with-gssapi \ --with-ldap \ diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index c62855d3089..72dba40551f 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -22,8 +22,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_legacy_string_agg \ gp_replica_check \ gp_toolkit \ - pg_hint_plan \ - diskquota + pg_hint_plan else recurse_targets = gp_sparse_vector \ gp_distribution_policy \ @@ -31,8 +30,11 @@ else gp_legacy_string_agg \ gp_exttable_fdw \ gp_toolkit \ - pg_hint_plan \ - diskquota + pg_hint_plan +endif + +ifeq "$(with_diskquota)" "yes" + recurse_targets += diskquota endif ifeq "$(with_gp_stats_collector)" "yes" @@ -105,4 +107,4 @@ installcheck: $(MAKE) -C gp_sparse_vector installcheck $(MAKE) -C gp_toolkit installcheck $(MAKE) -C gp_exttable_fdw installcheck - $(MAKE) -C diskquota installcheck + if [ "$(with_diskquota)" = "yes" ]; then $(MAKE) -C diskquota installcheck; fi From a130fd97b1e24179960ed21a6c5ec4f203c24718 Mon Sep 17 00:00:00 2001 From: FairyFar Date: Thu, 9 Apr 2026 20:17:24 +0800 Subject: [PATCH 114/167] Improve the SQL tab-completion feature for resource group (#1669) 1. For the time being, we will not handle the tab-completion feature of resource queue because there is a trend for resource group to replace resource queue. 2. Correct the description of the gp_resource_manager parameter. --- src/backend/utils/misc/guc_gp.c | 2 +- src/bin/psql/tab-complete.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 42b61dfbbbb..f846cc24aa3 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -4936,7 +4936,7 @@ struct config_string ConfigureNamesString_gp[] = { {"gp_resource_manager", PGC_POSTMASTER, RESOURCES, gettext_noop("Sets the type of resource manager."), - gettext_noop("Only support \"queue\" and \"group\" for now.") + gettext_noop("Only support \"queue\", \"group\" and \"group-v2\" for now.") }, &gp_resource_manager_str, "queue", diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index dc401f503b1..905ad740555 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1614,6 +1614,12 @@ psql_completion(const char *text, int start, int end) NULL }; + static const char *const list_resource_group_type[] = { + "CONCURRENCY", "CPU_MAX_PERCENT", "CPUSET", "CPU_WEIGHT", + "MEMORY_QUOTA", "MIN_COST", "IO_LIMIT", + NULL + }; + /* * Temporary workaround for a bug in recent (2019) libedit: it incorrectly * de-escapes the input "text", causing us to fail to recognize backslash @@ -3115,8 +3121,8 @@ psql_completion(const char *text, int start, int end) else if (Matches("CREATE", "ROLE|USER|GROUP", MatchAny, "IN")) COMPLETE_WITH("GROUP", "ROLE"); -/* CREATE/DROP RESOURCE GROUP/QUEUE */ - else if (Matches("CREATE|DROP", "RESOURCE")) +/* CREATE/DROP/ALTER RESOURCE GROUP/QUEUE */ + else if (Matches("CREATE|DROP|ALTER", "RESOURCE")) { static const char *const list_CREATERESOURCEGROUP[] = {"GROUP", "QUEUE", NULL}; @@ -3130,19 +3136,19 @@ psql_completion(const char *text, int start, int end) else if (Matches("CREATE", "PROFILE", MatchAny, "LIMIT")) COMPLETE_WITH("FAILED_LOGIN_ATTEMPTS", "PASSWORD_REUSE_MAX", "PASSWORD_LOCK_TIME"); - /* CREATE/DROP RESOURCE GROUP */ - else if (TailMatches("CREATE|DROP", "RESOURCE", "GROUP")) + /* CREATE/DROP/ALTER RESOURCE GROUP */ + else if (TailMatches("CREATE|DROP|ALTER", "RESOURCE", "GROUP")) COMPLETE_WITH_QUERY(Query_for_list_of_resgroups); /* CREATE RESOURCE GROUP */ else if (TailMatches("CREATE|DROP", "RESOURCE", "GROUP", MatchAny)) COMPLETE_WITH("WITH ("); + /* ALTER RESOURCE GROUP */ + else if (TailMatches("ALTER", "RESOURCE", "GROUP", MatchAny)) + COMPLETE_WITH("SET"); + else if (TailMatches("ALTER", "RESOURCE", "GROUP", MatchAny, "SET")) + COMPLETE_WITH_LIST(list_resource_group_type); else if (TailMatches("RESOURCE", "GROUP", MatchAny, "WITH", "(")) - { - static const char *const list_CREATERESOURCEGROUP[] = - {"CONCURRENCY", "CPU_MAX_PERCENT", "CPUSET", "CPU_WEIGHT", "MEMORY_QUOTA", "MIN_COST", "IO_LIMIT", NULL}; - - COMPLETE_WITH_LIST(list_CREATERESOURCEGROUP); - } + COMPLETE_WITH_LIST(list_resource_group_type); /* CREATE TYPE */ else if (Matches("CREATE", "TYPE", MatchAny)) From bf1e6be97857eb9f1927859e2762f2e37b5c39aa Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 8 Apr 2026 22:47:38 +0800 Subject: [PATCH 115/167] Fix aoco_relation_size() using wrong snapshot to read pg_aocsseg aoco_relation_size() used GetLatestSnapshot() to read pg_aocsseg catalog metadata. During ALTER TABLE SET DISTRIBUTED BY on AOCO tables, the reader gang's GetLatestSnapshot() cannot see pg_aocsseg rows written by the writer gang within the same distributed transaction (uncommitted local xid), causing the function to return 0 bytes. This led to relpages=0 being passed to vac_update_relstats() alongside a non-zero totalrows from sampling (which correctly uses GetCatalogSnapshot()), triggering an assertion failure: FailedAssertion: "Gp_role == GP_ROLE_UTILITY", vacuum.c:1738 Fix by passing NULL to GetAllAOCSFileSegInfo() so that systable_beginscan() uses GetCatalogSnapshot() internally, consistent with appendonly_relation_size() for AO row tables. --- src/backend/access/aocs/aocsam_handler.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/backend/access/aocs/aocsam_handler.c b/src/backend/access/aocs/aocsam_handler.c index c2faa538e10..4b3cd2a52ef 100644 --- a/src/backend/access/aocs/aocsam_handler.c +++ b/src/backend/access/aocs/aocsam_handler.c @@ -2044,15 +2044,25 @@ static uint64 aoco_relation_size(Relation rel, ForkNumber forkNumber) { AOCSFileSegInfo **allseg; - Snapshot snapshot; uint64 totalbytes = 0; int totalseg; if (forkNumber != MAIN_FORKNUM) return totalbytes; - snapshot = RegisterSnapshot(GetLatestSnapshot()); - allseg = GetAllAOCSFileSegInfo(rel, snapshot, &totalseg, NULL); + /* + * Pass NULL as snapshot so that GetAllAOCSFileSegInfo -> systable_beginscan + * uses GetCatalogSnapshot() internally. This is consistent with + * appendonly_relation_size() for AO row tables and ensures pg_aocsseg + * entries are visible even when called within the same transaction that + * populated them (e.g. ALTER TABLE SET DISTRIBUTED BY). + * + * Using GetLatestSnapshot() here previously caused the metadata to be + * invisible on QE segments during in-transaction redistribution, leading + * to a zero return value and a subsequent assertion failure in + * vac_update_relstats(). + */ + allseg = GetAllAOCSFileSegInfo(rel, NULL, &totalseg, NULL); for (int seg = 0; seg < totalseg; seg++) { for (int attr = 0; attr < RelationGetNumberOfAttributes(rel); attr++) @@ -2079,7 +2089,6 @@ aoco_relation_size(Relation rel, ForkNumber forkNumber) FreeAllAOCSSegFileInfo(allseg, totalseg); pfree(allseg); } - UnregisterSnapshot(snapshot); return totalbytes; } From 2414aa482ab0c054cc1adb80ea558ba3c35c244e Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Fri, 10 Apr 2026 06:23:35 -0700 Subject: [PATCH 116/167] CI: fix 'Check and Display Regression Diffs' step to use bash shell (#1673) * CI: fix 'Check and Display Regression Diffs' step to use bash shell The step used bash-specific [[ ]] syntax but lacked shell: bash {0}, causing failures when the runner defaulted to sh. --- .github/workflows/build-cloudberry-rocky8.yml | 1 + .github/workflows/build-cloudberry.yml | 1 + .github/workflows/build-deb-cloudberry.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 433363d6bc6..6665be6825e 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -1677,6 +1677,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 4cff875836f..5e0fbc1aa02 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -1681,6 +1681,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 5d458c46e13..f865eacba40 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -1619,6 +1619,7 @@ jobs: - name: Check and Display Regression Diffs if: always() + shell: bash {0} run: | # Search for regression.diffs recursively found_file=$(find . -type f -name "regression.diffs" | head -n 1) From 6e072e5fc4196f144b5380887e1449e15a49e414 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Thu, 2 Apr 2026 18:21:09 +0800 Subject: [PATCH 117/167] Fix SIGSEGV in fsm_extend when vacuuming tables in non-default tablespace The commit df1e2ff ("Prevent CREATE TABLE from using dangling tablespace") added a call to TablespaceLockTuple() inside TablespaceCreateDbspace() for every non-default tablespace, including the common path where the per-database directory already exists. That lock acquisition calls LockSharedObject(), which calls AcceptInvalidationMessages(), allowing pending sinval messages to be processed at an unexpected point deep inside smgrcreate(). This creates a crash window during VACUUM of a heap table (including auxiliary tables such as the aoseg table of an AO relation) that lives in a non-default tablespace and has never been vacuumed before (so no FSM or VM fork exists yet): lazy_scan_heap -> visibilitymap_pin -> vm_readbuf -> vm_extend smgrcreate(VM fork) + CacheInvalidateSmgr() <- queues SHAREDINVALSMGR_ID -> RecordPageWithFreeSpace -> fsm_readbuf -> fsm_extend smgrcreate(FSM fork) -> mdcreate -> TablespaceCreateDbspace [CBDB-specific for non-default tablespaces] -> TablespaceLockTuple -> LockSharedObject -> AcceptInvalidationMessages() -> smgrclosenode() -> rel->rd_smgr = NULL rel->rd_smgr->smgr_cached_nblocks[FSM_FORKNUM] = ... -> SIGSEGV (NULL dereference) at freespace.c:637 Fix: add RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() (freespace.c) and vm_extend() (visibilitymap.c). RelationOpenSmgr is a no-op when rd_smgr is still valid, and re-opens the smgr handle if the sinval handler has closed it. The identical guard already exists earlier in both functions for the LockRelationForExtension path. Add a regression test (vacuum_fsm_nondefault_tablespace) covering both a plain heap table and an AO table in a non-default tablespace. --- src/backend/access/heap/visibilitymap.c | 8 +++ src/backend/storage/freespace/freespace.c | 8 +++ src/test/regress/expected/.gitignore | 1 + src/test/regress/greenplum_schedule | 1 + .../vacuum_fsm_nondefault_tablespace.source | 54 ++++++++++++++ .../vacuum_fsm_nondefault_tablespace.source | 70 +++++++++++++++++++ src/test/regress/sql/.gitignore | 1 + 7 files changed, 143 insertions(+) create mode 100644 src/test/regress/input/vacuum_fsm_nondefault_tablespace.source create mode 100644 src/test/regress/output/vacuum_fsm_nondefault_tablespace.source diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index e198df65d82..7d252fa94ab 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -645,6 +645,14 @@ vm_extend(Relation rel, BlockNumber vm_nblocks) !smgrexists(rel->rd_smgr, VISIBILITYMAP_FORKNUM)) smgrcreate(rel->rd_smgr, VISIBILITYMAP_FORKNUM, false); + /* + * Might have to re-open if smgrcreate triggered AcceptInvalidationMessages + * (via TablespaceCreateDbspace -> LockSharedObject for non-default + * tablespaces), which may have processed a pending SHAREDINVALSMGR_ID + * message and closed our smgr entry. + */ + RelationOpenSmgr(rel); + /* Invalidate cache so that smgrnblocks() asks the kernel. */ rel->rd_smgr->smgr_cached_nblocks[VISIBILITYMAP_FORKNUM] = InvalidBlockNumber; vm_nblocks_now = smgrnblocks(rel->rd_smgr, VISIBILITYMAP_FORKNUM); diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index 796b915156b..ee97e757115 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -633,6 +633,14 @@ fsm_extend(Relation rel, BlockNumber fsm_nblocks) !smgrexists(rel->rd_smgr, FSM_FORKNUM)) smgrcreate(rel->rd_smgr, FSM_FORKNUM, false); + /* + * Might have to re-open if smgrcreate triggered AcceptInvalidationMessages + * (via TablespaceCreateDbspace -> LockSharedObject for non-default + * tablespaces), which may have processed a pending SHAREDINVALSMGR_ID + * message and closed our smgr entry. + */ + RelationOpenSmgr(rel); + /* Invalidate cache so that smgrnblocks() asks the kernel. */ rel->rd_smgr->smgr_cached_nblocks[FSM_FORKNUM] = InvalidBlockNumber; fsm_nblocks_now = smgrnblocks(rel->rd_smgr, FSM_FORKNUM); diff --git a/src/test/regress/expected/.gitignore b/src/test/regress/expected/.gitignore index c837ca324d5..f625061dbd6 100644 --- a/src/test/regress/expected/.gitignore +++ b/src/test/regress/expected/.gitignore @@ -70,3 +70,4 @@ /ao_unique_index_partition.out /bfv_copy.out /copy_encoding_error.out +/vacuum_fsm_nondefault_tablespace.out diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 3c8f7965b28..604616791c8 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -167,6 +167,7 @@ test: instr_in_shmem_verify # hold locks. test: partition_locking test: vacuum_gp +test: vacuum_fsm_nondefault_tablespace test: resource_queue_stat # background analyze may affect pgstat test: pg_stat diff --git a/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source b/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source new file mode 100644 index 00000000000..adce9ab77de --- /dev/null +++ b/src/test/regress/input/vacuum_fsm_nondefault_tablespace.source @@ -0,0 +1,54 @@ +-- Test: VACUUM on a table in a non-default tablespace does not crash on first run. +-- +-- Bug: SIGSEGV in fsm_extend() (freespace.c:637) when vacuuming a heap table +-- (or an AO table's aoseg auxiliary table) that resides in a non-default +-- tablespace for the very first time. +-- +-- Root cause: commit "Prevent CREATE TABLE from using dangling tablespace" +-- added TablespaceLockTuple() in TablespaceCreateDbspace() for non-default +-- tablespaces. That call reaches AcceptInvalidationMessages() via +-- LockSharedObject(), which processes a pending SHAREDINVALSMGR_ID message +-- that vm_extend() had queued via CacheInvalidateSmgr(), nullifying +-- rel->rd_smgr before fsm_extend() dereferences it at freespace.c:637. +-- +-- Fix: added RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() +-- (freespace.c) and vm_extend() (visibilitymap.c) so that rd_smgr is +-- re-opened if the sinval handler closed it. + +CREATE TABLESPACE fsm_ts_test LOCATION '@testtablespace@'; + +-- Case 1: plain heap table in non-default tablespace, first VACUUM. +-- Before the fix this crashed with SIGSEGV at freespace.c:637: +-- vm_extend() -> CacheInvalidateSmgr (queues SHAREDINVALSMGR_ID) +-- fsm_extend() -> smgrcreate -> TablespaceCreateDbspace +-- -> TablespaceLockTuple -> AcceptInvalidationMessages +-- -> processes SHAREDINVALSMGR_ID -> rel->rd_smgr = NULL +-- -> rel->rd_smgr->smgr_cached_nblocks[...] = ... SIGSEGV +CREATE TABLE fsm_ts_heap (id int, val text) + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_heap SELECT i, repeat('x', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; +-- Second VACUUM must also succeed (FSM/VM now exist, different code path). +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; +DROP TABLE fsm_ts_heap; + +-- Case 2: AO table in non-default tablespace. +-- The crash occurs inside the recursive vacuum of the aoseg auxiliary table +-- (which is also a heap table stored in the same non-default tablespace). +CREATE TABLE fsm_ts_ao (id int, val text) + USING ao_row + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_ao SELECT i, repeat('y', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; +-- Second VACUUM must also succeed. +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; +DROP TABLE fsm_ts_ao; + +-- Cleanup. +DROP TABLESPACE fsm_ts_test; diff --git a/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source b/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source new file mode 100644 index 00000000000..fc2ff245691 --- /dev/null +++ b/src/test/regress/output/vacuum_fsm_nondefault_tablespace.source @@ -0,0 +1,70 @@ +-- Test: VACUUM on a table in a non-default tablespace does not crash on first run. +-- +-- Bug: SIGSEGV in fsm_extend() (freespace.c:637) when vacuuming a heap table +-- (or an AO table's aoseg auxiliary table) that resides in a non-default +-- tablespace for the very first time. +-- +-- Root cause: commit "Prevent CREATE TABLE from using dangling tablespace" +-- added TablespaceLockTuple() in TablespaceCreateDbspace() for non-default +-- tablespaces. That call reaches AcceptInvalidationMessages() via +-- LockSharedObject(), which processes a pending SHAREDINVALSMGR_ID message +-- that vm_extend() had queued via CacheInvalidateSmgr(), nullifying +-- rel->rd_smgr before fsm_extend() dereferences it at freespace.c:637. +-- +-- Fix: added RelationOpenSmgr(rel) after smgrcreate() in both fsm_extend() +-- (freespace.c) and vm_extend() (visibilitymap.c) so that rd_smgr is +-- re-opened if the sinval handler closed it. +CREATE TABLESPACE fsm_ts_test LOCATION '@testtablespace@'; +-- Case 1: plain heap table in non-default tablespace, first VACUUM. +-- Before the fix this crashed with SIGSEGV at freespace.c:637: +-- vm_extend() -> CacheInvalidateSmgr (queues SHAREDINVALSMGR_ID) +-- fsm_extend() -> smgrcreate -> TablespaceCreateDbspace +-- -> TablespaceLockTuple -> AcceptInvalidationMessages +-- -> processes SHAREDINVALSMGR_ID -> rel->rd_smgr = NULL +-- -> rel->rd_smgr->smgr_cached_nblocks[...] = ... SIGSEGV +CREATE TABLE fsm_ts_heap (id int, val text) + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_heap SELECT i, repeat('x', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; + count +------- + 500 +(1 row) + +-- Second VACUUM must also succeed (FSM/VM now exist, different code path). +VACUUM ANALYZE fsm_ts_heap; +SELECT count(*) FROM fsm_ts_heap; + count +------- + 500 +(1 row) + +DROP TABLE fsm_ts_heap; +-- Case 2: AO table in non-default tablespace. +-- The crash occurs inside the recursive vacuum of the aoseg auxiliary table +-- (which is also a heap table stored in the same non-default tablespace). +CREATE TABLE fsm_ts_ao (id int, val text) + USING ao_row + TABLESPACE fsm_ts_test + DISTRIBUTED BY (id); +INSERT INTO fsm_ts_ao SELECT i, repeat('y', 80) FROM generate_series(1, 500) i; +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; + count +------- + 500 +(1 row) + +-- Second VACUUM must also succeed. +VACUUM ANALYZE fsm_ts_ao; +SELECT count(*) FROM fsm_ts_ao; + count +------- + 500 +(1 row) + +DROP TABLE fsm_ts_ao; +-- Cleanup. +DROP TABLESPACE fsm_ts_test; diff --git a/src/test/regress/sql/.gitignore b/src/test/regress/sql/.gitignore index 9b5f3660fa7..3a340338616 100644 --- a/src/test/regress/sql/.gitignore +++ b/src/test/regress/sql/.gitignore @@ -64,3 +64,4 @@ /ao_unique_index_partition.sql /bfv_copy.sql /copy_encoding_error.sql +/vacuum_fsm_nondefault_tablespace.sql From 6756a1cd355b8a84a305d6f22fa59d48aa86e062 Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Thu, 16 Apr 2026 10:16:19 -0700 Subject: [PATCH 118/167] ORCA: fall back to Postgres planner for KNN ORDER BY queries (#1653) ORCA is unaware of amcanorderbyop, so it plans "ORDER BY col <-> val" queries with a full Seq Scan + Sort instead of a native KNN ordered index scan. Detect this pattern by checking whether any ORDER BY target is an operator with amoppurpose = AMOP_ORDER in pg_amop and at least one direct Var argument, then raise ExmiQuery2DXLUnsupportedFeature to hand the query off to the Postgres planner, which generates an efficient Index Only Scan with native KNN ordering. Queries where the ordering operator's arguments are entirely computed expressions (e.g. circle(col,1) <-> point(0,0)) are excluded from the fallback to avoid lossy-distance errors in index-only scans. Co-authored-by: reshke --- .../btree_gist/expected/cash_optimizer.out | 11 +- .../btree_gist/expected/date_optimizer.out | 7 +- .../btree_gist/expected/float4_optimizer.out | 7 +- .../btree_gist/expected/float8_optimizer.out | 7 +- .../btree_gist/expected/int2_optimizer.out | 7 +- .../btree_gist/expected/int4_optimizer.out | 7 +- .../btree_gist/expected/int8_optimizer.out | 7 +- .../expected/interval_optimizer.out | 24 ++-- .../btree_gist/expected/time_optimizer.out | 7 +- .../expected/timestamp_optimizer.out | 7 +- .../expected/timestamptz_optimizer.out | 7 +- .../pg_trgm/expected/pg_trgm_optimizer.out | 23 ++-- src/backend/gpopt/gpdbwrappers.cpp | 11 ++ .../gpopt/translate/CTranslatorQueryToDXL.cpp | 9 ++ src/backend/optimizer/util/walkers.c | 112 ++++++++++++++++++ src/include/gpopt/gpdbwrappers.h | 3 + src/include/optimizer/walkers.h | 1 + .../expected/create_index_optimizer.out | 45 ++++--- src/test/regress/expected/gist_optimizer.out | 63 +++++----- 19 files changed, 242 insertions(+), 123 deletions(-) diff --git a/contrib/btree_gist/expected/cash_optimizer.out b/contrib/btree_gist/expected/cash_optimizer.out index 171dec7e511..f2c9ac07420 100644 --- a/contrib/btree_gist/expected/cash_optimizer.out +++ b/contrib/btree_gist/expected/cash_optimizer.out @@ -77,12 +77,11 @@ SELECT a, a <-> '21472.79' FROM moneytmp ORDER BY a <-> '21472.79' LIMIT 3; QUERY PLAN ------------------------------------------------------------ Limit - -> Sort - Sort Key: ((a <-> '$21,472.79'::money)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Seq Scan on moneytmp - Optimizer: GPORCA + -> Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((a <-> '$21,472.79'::money)) + -> Limit + -> Index Only Scan using moneyidx on moneytmp + Order By: (a <-> '$21,472.79'::money) (7 rows) SELECT a, a <-> '21472.79' FROM moneytmp ORDER BY a <-> '21472.79' LIMIT 3; diff --git a/contrib/btree_gist/expected/date_optimizer.out b/contrib/btree_gist/expected/date_optimizer.out index a77041f847f..12269cf169b 100644 --- a/contrib/btree_gist/expected/date_optimizer.out +++ b/contrib/btree_gist/expected/date_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '2001-02-13' FROM datetmp ORDER BY a <-> '2001-02-13' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '02-13-2001'::date)) - -> Sort - Sort Key: ((a <-> '02-13-2001'::date)) - -> Seq Scan on datetmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using dateidx on datetmp + Order By: (a <-> '02-13-2001'::date) (7 rows) SELECT a, a <-> '2001-02-13' FROM datetmp ORDER BY a <-> '2001-02-13' LIMIT 3; diff --git a/contrib/btree_gist/expected/float4_optimizer.out b/contrib/btree_gist/expected/float4_optimizer.out index cc40e9bd1ae..7b71a2f5112 100644 --- a/contrib/btree_gist/expected/float4_optimizer.out +++ b/contrib/btree_gist/expected/float4_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '-179'::real)) - -> Sort - Sort Key: ((a <-> '-179'::real)) - -> Seq Scan on float4tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using float4idx on float4tmp + Order By: (a <-> '-179'::real) (7 rows) SELECT a, a <-> '-179.0' FROM float4tmp ORDER BY a <-> '-179.0' LIMIT 3; diff --git a/contrib/btree_gist/expected/float8_optimizer.out b/contrib/btree_gist/expected/float8_optimizer.out index 1bd96c44d3b..18e5c195286 100644 --- a/contrib/btree_gist/expected/float8_optimizer.out +++ b/contrib/btree_gist/expected/float8_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '-1890'::double precision)) - -> Sort - Sort Key: ((a <-> '-1890'::double precision)) - -> Seq Scan on float8tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using float8idx on float8tmp + Order By: (a <-> '-1890'::double precision) (7 rows) SELECT a, a <-> '-1890.0' FROM float8tmp ORDER BY a <-> '-1890.0' LIMIT 3; diff --git a/contrib/btree_gist/expected/int2_optimizer.out b/contrib/btree_gist/expected/int2_optimizer.out index fdfc859097b..f8f6a428b93 100644 --- a/contrib/btree_gist/expected/int2_optimizer.out +++ b/contrib/btree_gist/expected/int2_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '237' FROM int2tmp ORDER BY a <-> '237' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '237'::smallint)) - -> Sort - Sort Key: ((a <-> '237'::smallint)) - -> Seq Scan on int2tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int2idx on int2tmp + Order By: (a <-> '237'::smallint) (7 rows) SELECT a, a <-> '237' FROM int2tmp ORDER BY a <-> '237' LIMIT 3; diff --git a/contrib/btree_gist/expected/int4_optimizer.out b/contrib/btree_gist/expected/int4_optimizer.out index 67107e63bfa..6877fb09af5 100644 --- a/contrib/btree_gist/expected/int4_optimizer.out +++ b/contrib/btree_gist/expected/int4_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '237' FROM int4tmp ORDER BY a <-> '237' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 237)) - -> Sort - Sort Key: ((a <-> 237)) - -> Seq Scan on int4tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int4idx on int4tmp + Order By: (a <-> 237) (7 rows) SELECT a, a <-> '237' FROM int4tmp ORDER BY a <-> '237' LIMIT 3; diff --git a/contrib/btree_gist/expected/int8_optimizer.out b/contrib/btree_gist/expected/int8_optimizer.out index ba8e21135e8..962dd314661 100644 --- a/contrib/btree_gist/expected/int8_optimizer.out +++ b/contrib/btree_gist/expected/int8_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '464571291354841' FROM int8tmp ORDER BY a <-> '464571291354841' Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '464571291354841'::bigint)) - -> Sort - Sort Key: ((a <-> '464571291354841'::bigint)) - -> Seq Scan on int8tmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using int8idx on int8tmp + Order By: (a <-> '464571291354841'::bigint) (7 rows) SELECT a, a <-> '464571291354841' FROM int8tmp ORDER BY a <-> '464571291354841' LIMIT 3; diff --git a/contrib/btree_gist/expected/interval_optimizer.out b/contrib/btree_gist/expected/interval_optimizer.out index f5afd17456b..f0a4e850aeb 100644 --- a/contrib/btree_gist/expected/interval_optimizer.out +++ b/contrib/btree_gist/expected/interval_optimizer.out @@ -74,15 +74,15 @@ SELECT count(*) FROM intervaltmp WHERE a > '199 days 21:21:23'::interval; EXPLAIN (COSTS OFF) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Sort - Sort Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Seq Scan on intervaltmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using intervalidx on intervaltmp + Order By: (a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval) + Optimizer: Postgres query optimizer (7 rows) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; @@ -96,15 +96,15 @@ SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21 SET enable_indexonlyscan=off; EXPLAIN (COSTS OFF) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; - QUERY PLAN ------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Sort - Sort Key: ((a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval)) - -> Seq Scan on intervaltmp - Optimizer: GPORCA + -> Limit + -> Index Scan using intervalidx on intervaltmp + Order By: (a <-> '@ 199 days 21 hours 21 mins 23 secs'::interval) + Optimizer: Postgres query optimizer (7 rows) SELECT a, a <-> '199 days 21:21:23' FROM intervaltmp ORDER BY a <-> '199 days 21:21:23' LIMIT 3; diff --git a/contrib/btree_gist/expected/time_optimizer.out b/contrib/btree_gist/expected/time_optimizer.out index 590ada880b9..40d49e79b02 100644 --- a/contrib/btree_gist/expected/time_optimizer.out +++ b/contrib/btree_gist/expected/time_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '10:57:11' FROM timetmp ORDER BY a <-> '10:57:11' LIMIT 3; Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> '10:57:11'::time without time zone)) - -> Sort - Sort Key: ((a <-> '10:57:11'::time without time zone)) - -> Seq Scan on timetmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timeidx on timetmp + Order By: (a <-> '10:57:11'::time without time zone) (7 rows) SELECT a, a <-> '10:57:11' FROM timetmp ORDER BY a <-> '10:57:11' LIMIT 3; diff --git a/contrib/btree_gist/expected/timestamp_optimizer.out b/contrib/btree_gist/expected/timestamp_optimizer.out index 1b8e709fe90..85c3a1a5e5d 100644 --- a/contrib/btree_gist/expected/timestamp_optimizer.out +++ b/contrib/btree_gist/expected/timestamp_optimizer.out @@ -79,10 +79,9 @@ SELECT a, a <-> '2004-10-26 08:55:08' FROM timestamptmp ORDER BY a <-> '2004-10- Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone)) - -> Sort - Sort Key: ((a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone)) - -> Seq Scan on timestamptmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timestampidx on timestamptmp + Order By: (a <-> 'Tue Oct 26 08:55:08 2004'::timestamp without time zone) (7 rows) SELECT a, a <-> '2004-10-26 08:55:08' FROM timestamptmp ORDER BY a <-> '2004-10-26 08:55:08' LIMIT 3; diff --git a/contrib/btree_gist/expected/timestamptz_optimizer.out b/contrib/btree_gist/expected/timestamptz_optimizer.out index 2173c5dca35..a9e043f98a6 100644 --- a/contrib/btree_gist/expected/timestamptz_optimizer.out +++ b/contrib/btree_gist/expected/timestamptz_optimizer.out @@ -199,10 +199,9 @@ SELECT a, a <-> '2018-12-18 10:59:54 GMT+2' FROM timestamptztmp ORDER BY a <-> ' Limit -> Gather Motion 3:1 (slice1; segments: 3) Merge Key: ((a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone)) - -> Sort - Sort Key: ((a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone)) - -> Seq Scan on timestamptztmp - Optimizer: GPORCA + -> Limit + -> Index Only Scan using timestamptzidx on timestamptztmp + Order By: (a <-> 'Tue Dec 18 04:59:54 2018 PST'::timestamp with time zone) (7 rows) SELECT a, a <-> '2018-12-18 10:59:54 GMT+2' FROM timestamptztmp ORDER BY a <-> '2018-12-18 10:59:54 GMT+2' LIMIT 3; diff --git a/contrib/pg_trgm/expected/pg_trgm_optimizer.out b/contrib/pg_trgm/expected/pg_trgm_optimizer.out index 4597b8ca047..a1e9b3d299d 100644 --- a/contrib/pg_trgm/expected/pg_trgm_optimizer.out +++ b/contrib/pg_trgm/expected/pg_trgm_optimizer.out @@ -2351,6 +2351,7 @@ select t <-> 'q0987wertyu0988', t from test_trgm order by t <-> 'q0987wertyu0988 -> Limit -> Index Scan using trgm_idx on test_trgm Order By: (t <-> 'q0987wertyu0988'::text) + Optimizer: Postgres query optimizer (7 rows) select t <-> 'q0987wertyu0988', t from test_trgm order by t <-> 'q0987wertyu0988' limit 2; @@ -5003,8 +5004,8 @@ select * from test2 where t ~ '/\d+/-\d'; -- test = operator explain (costs off) select * from test2 where t = 'abcdef'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = 'abcdef'::text) @@ -5020,8 +5021,8 @@ select * from test2 where t = 'abcdef'; explain (costs off) select * from test2 where t = '%line%'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------ Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = '%line%'::text) @@ -5311,14 +5312,15 @@ select * from test2 where t ~ '/\d+/-\d'; -- test = operator explain (costs off) select * from test2 where t = 'abcdef'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = 'abcdef'::text) -> Bitmap Index Scan on test2_idx_gist Index Cond: (t = 'abcdef'::text) -(2 rows) + Optimizer: Postgres query optimizer +(6 rows) select * from test2 where t = 'abcdef'; t @@ -5328,13 +5330,14 @@ select * from test2 where t = 'abcdef'; explain (costs off) select * from test2 where t = '%line%'; - QUERY PLAN ------------------------------------------- + QUERY PLAN +------------------------------------------------- Gather Motion 1:1 (slice1; segments: 1) -> Bitmap Heap Scan on test2 Recheck Cond: (t = '%line%'::text) -> Bitmap Index Scan on test2_idx_gist Index Cond: (t = '%line%'::text) + Optimizer: Postgres query optimizer (6 rows) select * from test2 where t = '%line%'; @@ -5423,7 +5426,7 @@ SELECT DISTINCT city, similarity(city, 'Warsaw'), show_limit() -> Index Scan using restaurants_city_idx on restaurants Index Cond: (city % 'Warsaw'::text) Filter: (city % 'Warsaw'::text) - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (9 rows) SELECT set_limit(0.3); diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index aca95b2cc0a..4d8d8c59100 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -2012,6 +2012,17 @@ gpdb::CheckCollation(Node *node) return -1; } +bool +gpdb::HasOrderByOrderingOp(Query *query) +{ + GP_WRAP_START; + { + return has_orderby_ordering_op(query); + } + GP_WRAP_END; + return false; +} + Node * gpdb::CoerceToCommonType(ParseState *pstate, Node *node, Oid target_type, const char *context) diff --git a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp index 20cc6557c28..99d87917b38 100644 --- a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp @@ -324,6 +324,15 @@ CTranslatorQueryToDXL::CheckUnsupportedNodeTypes(Query *query) GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Non-default collation")); } + + // ORCA does not support amcanorderbyop (KNN ordered index scans). + // Fall back to the PostgreSQL planner for queries whose ORDER BY + // contains an ordering operator (e.g., <-> for distance). + if (gpdb::HasOrderByOrderingOp(query)) + { + GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, + GPOS_WSZ_LIT("ORDER BY with ordering operator (amcanorderbyop)")); + } } //--------------------------------------------------------------------------- diff --git a/src/backend/optimizer/util/walkers.c b/src/backend/optimizer/util/walkers.c index 3b3d0311d06..be806f4daf7 100644 --- a/src/backend/optimizer/util/walkers.c +++ b/src/backend/optimizer/util/walkers.c @@ -8,11 +8,17 @@ #include "postgres.h" +#include "access/htup_details.h" +#include "catalog/pg_amop.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" +#include "optimizer/optimizer.h" #include "optimizer/walkers.h" +#include "utils/catcache.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" /** * Plan node walker related methods. @@ -1011,3 +1017,109 @@ check_collation_walker(Node *node, check_collation_context *context) } } +/* + * is_ordering_op + * + * Return true if the operator is registered as an ordering operator + * (amoppurpose = AMOP_ORDER) in any opfamily in pg_amop. + */ +static bool +is_ordering_op(Oid opno) +{ + CatCList *catlist = SearchSysCacheList1(AMOPOPID, + ObjectIdGetDatum(opno)); + + for (int i = 0; i < catlist->n_members; i++) + { + HeapTuple tp = &catlist->members[i]->tuple; + Form_pg_amop amop = (Form_pg_amop) GETSTRUCT(tp); + + if (amop->amoppurpose == AMOP_ORDER) + { + ReleaseSysCacheList(catlist); + return true; + } + } + ReleaseSysCacheList(catlist); + return false; +} + +/* + * has_plain_var_arg + * + * Return true if the OpExpr has at least one direct Var argument + * (not wrapped in a function or other expression). + * + * Implicit coercions such as RelabelType (binary-compatible casts, e.g. + * varchar -> text) are stripped before the check so that a column + * reference that was implicitly cast to match the operator's input type + * is still recognised as a plain Var. + */ +static bool +has_plain_var_arg(OpExpr *op) +{ + ListCell *arg_lc; + + foreach(arg_lc, op->args) + { + Node *arg = strip_implicit_coercions(lfirst(arg_lc)); + + if (IsA(arg, Var)) + return true; + } + return false; +} + +/* + * has_orderby_ordering_op + * + * Check if the query's ORDER BY uses ordering operators (amoppurpose = + * AMOP_ORDER in pg_amop) that the PostgreSQL planner can safely optimize + * with KNN-GiST index scans but ORCA cannot. + * + * Return true only when ALL ordering-operator expressions in ORDER BY + * have at least one direct Var (column reference) argument. Expressions + * like "circle(p,1) <-> point(0,0)" wrap the column in a function, + * which can cause "lossy distance functions are not supported in + * index-only scans" errors in the planner. In such cases we leave the + * query for ORCA to handle via Seq Scan + Sort. + */ +bool +has_orderby_ordering_op(Query *query) +{ + ListCell *lc; + bool found_ordering_op = false; + + if (query->sortClause == NIL) + return false; + + foreach(lc, query->sortClause) + { + SortGroupClause *sgc = (SortGroupClause *) lfirst(lc); + TargetEntry *tle = get_sortgroupclause_tle(sgc, query->targetList); + Node *expr = (Node *) tle->expr; + + if (!IsA(expr, OpExpr)) + continue; + + OpExpr *opexpr = (OpExpr *) expr; + + if (!is_ordering_op(opexpr->opno)) + continue; + + /* + * Found an ordering operator. Check that at least one argument is + * a plain Var. If any ordering operator has only computed arguments + * (e.g., function calls wrapping columns), bail out immediately — + * falling back to the planner could produce lossy distance errors + * in index-only scans. + */ + found_ordering_op = true; + + if (!has_plain_var_arg(opexpr)) + return false; + } + + return found_ordering_op; +} + diff --git a/src/include/gpopt/gpdbwrappers.h b/src/include/gpopt/gpdbwrappers.h index 261cd28b5f0..9ef53169599 100644 --- a/src/include/gpopt/gpdbwrappers.h +++ b/src/include/gpopt/gpdbwrappers.h @@ -673,6 +673,9 @@ int FindNodes(Node *node, List *nodeTags); // look for nodes with non-default collation; returns 1 if any exist, -1 otherwise int CheckCollation(Node *node); +// check if ORDER BY uses an ordering operator (amcanorderbyop) unsupported by ORCA +bool HasOrderByOrderingOp(Query *query); + Node *CoerceToCommonType(ParseState *pstate, Node *node, Oid target_type, const char *context); diff --git a/src/include/optimizer/walkers.h b/src/include/optimizer/walkers.h index 6d0d38717f5..d29bc5551e8 100644 --- a/src/include/optimizer/walkers.h +++ b/src/include/optimizer/walkers.h @@ -43,5 +43,6 @@ extern List *extract_nodes_plan(Plan *pl, int nodeTag, bool descendIntoSubquerie extern List *extract_nodes_expression(Node *node, int nodeTag, bool descendIntoSubqueries); extern int find_nodes(Node *node, List *nodeTags); extern int check_collation(Node *node); +extern bool has_orderby_ordering_op(Query *query); #endif /* WALKERS_H_ */ diff --git a/src/test/regress/expected/create_index_optimizer.out b/src/test/regress/expected/create_index_optimizer.out index 65f5f92b8bd..aca6fbb1332 100644 --- a/src/test/regress/expected/create_index_optimizer.out +++ b/src/test/regress/expected/create_index_optimizer.out @@ -652,18 +652,16 @@ SELECT * FROM point_tblv WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; --SELECT * FROM point_tbl WHERE f1 IS NOT NULL ORDER BY f1 <-> '0,1'; EXPLAIN (COSTS OFF) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Result - -> Sort - Sort Key: ((f1 <-> '(0,1)'::point)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Index Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) - Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision) AND (f1 <@ '(10,10),(-10,-10)'::box)) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((point_tbl.f1 <-> '(0,1)'::point)) + -> Index Only Scan using gpointind on point_tbl + Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Order By: (f1 <-> '(0,1)'::point) + Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision)) + Optimizer: Postgres query optimizer +(7 rows) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 @@ -767,18 +765,19 @@ SET enable_indexscan = OFF; SET enable_bitmapscan = ON; EXPLAIN (COSTS OFF) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - Result + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((point_tbl.f1 <-> '(0,1)'::point)) -> Sort - Sort Key: ((f1 <-> '(0,1)'::point)) - -> Result - -> Gather Motion 3:1 (slice1; segments: 3) - -> Index Scan using gpointind on point_tbl - Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) - Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision) AND (f1 <@ '(10,10),(-10,-10)'::box)) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + Sort Key: ((point_tbl.f1 <-> '(0,1)'::point)) + -> Bitmap Heap Scan on point_tbl + Recheck Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Filter: ((f1 <> '(1e-300,-1e-300)'::point) AND ((f1 <-> '(0,0)'::point) <> 'Infinity'::double precision)) + -> Bitmap Index Scan on gpointind + Index Cond: (f1 <@ '(10,10),(-10,-10)'::box) + Optimizer: Postgres query optimizer +(10 rows) SELECT * FROM point_tblv WHERE f1 <@ '(-10,-10),(10,10)':: box ORDER BY f1 <-> '0,1'; f1 diff --git a/src/test/regress/expected/gist_optimizer.out b/src/test/regress/expected/gist_optimizer.out index e9020c5db70..abb8b5524cf 100644 --- a/src/test/regress/expected/gist_optimizer.out +++ b/src/test/regress/expected/gist_optimizer.out @@ -98,18 +98,15 @@ select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)); explain (costs off) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by p <-> point(0.201, 0.201); - QUERY PLAN ---------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: ((p <-> '(0.201,0.201)'::point)) - -> Sort - Sort Key: ((p <-> '(0.201,0.201)'::point)) - -> Index Scan using gist_tbl_point_index on gist_tbl - Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) - Filter: (p <@ '(0.5,0.5),(0,0)'::box) - Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 -(9 rows) + QUERY PLAN +-------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((p <-> '(0.201,0.201)'::point)) + -> Index Only Scan using gist_tbl_point_index on gist_tbl + Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) + Order By: (p <-> '(0.201,0.201)'::point) + Optimizer: Postgres query optimizer +(6 rows) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by p <-> point(0.201, 0.201); @@ -132,18 +129,15 @@ order by p <-> point(0.201, 0.201); explain (costs off) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by point(0.101, 0.101) <-> p; - QUERY PLAN ---------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: (('(0.101,0.101)'::point <-> p)) - -> Sort - Sort Key: (('(0.101,0.101)'::point <-> p)) - -> Index Scan using gist_tbl_point_index on gist_tbl - Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) - Filter: (p <@ '(0.5,0.5),(0,0)'::box) - Optimizer: Pivotal Optimizer (GPORCA) version 3.83.0 -(9 rows) + QUERY PLAN +-------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: (('(0.101,0.101)'::point <-> p)) + -> Index Only Scan using gist_tbl_point_index on gist_tbl + Index Cond: (p <@ '(0.5,0.5),(0,0)'::box) + Order By: (p <-> '(0.101,0.101)'::point) + Optimizer: Postgres query optimizer +(6 rows) select p from gist_tbl where p <@ box(point(0,0), point(0.5, 0.5)) order by point(0.101, 0.101) <-> p; @@ -248,18 +242,15 @@ select b from gist_tbl where b <@ box(point(5,5), point(6,6)); explain (costs off) select b from gist_tbl where b <@ box(point(5,5), point(6,6)) order by b <-> point(5.2, 5.91); - QUERY PLAN -------------------------------------------------------------------- - Result - -> Gather Motion 3:1 (slice1; segments: 3) - Merge Key: ((b <-> '(5.2,5.91)'::point)) - -> Sort - Sort Key: ((b <-> '(5.2,5.91)'::point)) - -> Index Scan using gist_tbl_box_index on gist_tbl - Index Cond: (b <@ '(6,6),(5,5)'::box) - Filter: (b <@ '(6,6),(5,5)'::box) - Optimizer: Pivotal Optimizer (GPORCA) -(9 rows) + QUERY PLAN +------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) + Merge Key: ((b <-> '(5.2,5.91)'::point)) + -> Index Only Scan using gist_tbl_box_index on gist_tbl + Index Cond: (b <@ '(6,6),(5,5)'::box) + Order By: (b <-> '(5.2,5.91)'::point) + Optimizer: Postgres query optimizer +(6 rows) select b from gist_tbl where b <@ box(point(5,5), point(6,6)) order by b <-> point(5.2, 5.91); From 5f69072198c3ab0020cd5d4489f3dbe12081ecc5 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 12 Nov 2025 14:28:23 +0800 Subject: [PATCH 119/167] Fix: Python build dependencies installation Improve wheel and cython dependency management in gpMgmt/bin/Makefile to handle Ubuntu 24.04's PEP 668 restrictions while maintaining compatibility with Rocky Linux and older Ubuntu versions. Changes: - Split wheel and cython dependency checks into separate commands - Add fallback to --break-system-packages flag for Ubuntu 24.04+ - Only install dependencies if not already present in the system - Maintain backward compatibility with existing build environments This resolves build failures on Ubuntu 24.04 where pip install --user is restricted by default, while preserving the existing behavior on Rocky Linux 8/9 and Ubuntu 20.04/22.04 systems. --- gpMgmt/bin/Makefile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index c5eb6ccba9c..7092700b784 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -111,8 +111,19 @@ download-python-deps: else \ echo "PyGreSQL-$(PYGRESQL_VERSION).tar.gz already exists, skipping download"; \ fi - # Install wheel and cython for PyYAML building - pip3 install --user wheel "cython<3.0.0" + # Install wheel and cython for PyYAML building (only if not exists) + @if python3 -c "import wheel" >/dev/null 2>&1; then \ + echo "wheel already exists, skipping installation"; \ + else \ + echo "Installing wheel..."; \ + pip3 install --user wheel 2>/dev/null || pip3 install --user --break-system-packages wheel; \ + fi + @if python3 -c "import cython" >/dev/null 2>&1; then \ + echo "cython already exists, skipping installation"; \ + else \ + echo "Installing cython..."; \ + pip3 install --user "cython<3.0.0" 2>/dev/null || pip3 install --user --break-system-packages "cython<3.0.0"; \ + fi # # PyGreSQL From fc329540d0da2e1f3c06b6d6dece355c1b5fe7db Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Mon, 20 Apr 2026 21:43:32 -0700 Subject: [PATCH 120/167] ORCA: add optimizer_use_streaming_hashagg GUC (#1681) ORCA unconditionally sets stream_safe=true for all local HashAggs in FLocalHashAggStreamSafe, so the existing gp_use_streaming_hashagg GUC (which is only read by the Postgres planner path in cdbgroupingpaths.c) has no effect when optimizer=on. There was no way to disable streaming hash agg for ORCA plans. Introduce optimizer_use_streaming_hashagg (default on) and wire it through the standard CConfigParamMapping path: map it (negated) to a new EopttraceDisableStreamingHashAgg traceflag, and check that traceflag in FLocalHashAggStreamSafe. When the GUC is off, ORCA emits a non-streaming Partial HashAggregate that spills to disk and fully deduplicates. --- src/backend/gpopt/config/CConfigParamMapping.cpp | 7 ++++++- .../src/translate/CTranslatorExprToDXLUtils.cpp | 3 ++- .../include/naucrates/traceflags/traceflags.h | 3 +++ src/backend/utils/misc/guc_gp.c | 11 +++++++++++ src/include/utils/guc.h | 1 + src/include/utils/unsync_guc_name.h | 1 + 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/backend/gpopt/config/CConfigParamMapping.cpp b/src/backend/gpopt/config/CConfigParamMapping.cpp index 603855c50ec..62255364ba8 100644 --- a/src/backend/gpopt/config/CConfigParamMapping.cpp +++ b/src/backend/gpopt/config/CConfigParamMapping.cpp @@ -331,7 +331,12 @@ CConfigParamMapping::SConfigMappingElem CConfigParamMapping::m_elements[] = { false, // m_negate_param GPOS_WSZ_LIT( "Enable create window hash agg")}, - + + {EopttraceDisableStreamingHashAgg, &optimizer_use_streaming_hashagg, + true, // m_negate_param + GPOS_WSZ_LIT( + "Disable streaming hash agg in ORCA-generated local partial aggregations.")}, + }; //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp index 27f5cb688fe..cf497223553 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXLUtils.cpp @@ -1296,7 +1296,8 @@ CTranslatorExprToDXLUtils::FLocalHashAggStreamSafe(CExpression *pexprAgg) // is a local hash aggregate and it generates duplicates (therefore safe to stream) return (COperator::EgbaggtypeLocal == popAgg->Egbaggtype()) && - popAgg->FGeneratesDuplicates(); + popAgg->FGeneratesDuplicates() && + !GPOS_FTRACE(EopttraceDisableStreamingHashAgg); } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h index 2e489f214e5..8a18ace986a 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h @@ -250,6 +250,9 @@ enum EOptTraceFlag // Use the all key exclude the non-fixed key in AGG pds EopttraceAggRRSExcludeNonFixedKey = 103053, + // Disable streaming hash agg in ORCA-generated local partial aggregations + EopttraceDisableStreamingHashAgg = 103054, + /////////////////////////////////////////////////////// ///////////////////// statistics flags //////////////// ////////////////////////////////////////////////////// diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index f846cc24aa3..7a4433cfa98 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -154,6 +154,7 @@ bool enable_parallel_dedup_semi_join = true; bool enable_parallel_dedup_semi_reverse_join = true; bool parallel_query_use_streaming_hashagg = false; bool gp_use_streaming_hashagg = true; +bool optimizer_use_streaming_hashagg = true; int gp_appendonly_insert_files = 0; int gp_appendonly_insert_files_tuples_range = 0; int gp_random_insert_segments = 0; @@ -1909,6 +1910,16 @@ struct config_bool ConfigureNamesBool_gp[] = true, NULL, NULL }, + { + {"optimizer_use_streaming_hashagg", PGC_USERSET, DEVELOPER_OPTIONS, + gettext_noop("Use streaming hash agg in ORCA-generated local partial hash aggregations."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &optimizer_use_streaming_hashagg, + true, NULL, NULL + }, + { {"gp_force_random_redistribution", PGC_USERSET, CUSTOM_OPTIONS, gettext_noop("Force redistribution of insert for randomly-distributed."), diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index aa34138a4b5..652e0b451f3 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -295,6 +295,7 @@ extern bool enable_parallel_dedup_semi_join; extern bool enable_parallel_dedup_semi_reverse_join; extern bool parallel_query_use_streaming_hashagg; extern bool gp_use_streaming_hashagg; +extern bool optimizer_use_streaming_hashagg; extern int gp_appendonly_insert_files; extern int gp_appendonly_insert_files_tuples_range; extern int gp_random_insert_segments; diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 55a81df5bae..85ecb3548e6 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -501,6 +501,7 @@ "optimizer_skew_factor", "optimizer_use_external_constant_expression_evaluation_for_ints", "optimizer_use_gpdb_allocators", + "optimizer_use_streaming_hashagg", "optimizer_xform_bind_threshold", "parallel_leader_participation", "parallel_query_use_streaming_hashagg", From ed98e6f04c38fb5b8666b997f2bfbb8cabff64f9 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 18:19:14 +0800 Subject: [PATCH 121/167] Align release artifact naming with ASF incubator conventions Use base version (without -rcN suffix) for release tarball filename to align with Apache incubator release conventions. Changes: - Modified TAR_NAME to use ${BASE_VERSION} instead of ${TAG} - Tarball filename now follows pattern: apache-cloudberry-${BASE_VERSION}-src.tar.gz - Example: apache-cloudberry-2.0.0-incubating-src.tar.gz (instead of apache-cloudberry-2.0.0-incubating-rc1-src.tar.gz) Benefits: - Enables direct 'svn mv' to release repository after voting without renaming artifacts - Aligns with Apache release best practices where RC identifiers are used only for Git tags and voting process, not in final artifact names - Eliminates need to regenerate .sha512 files during promotion - Maintains consistency between tarball filename and extracted directory name - Simplifies release manager workflow The extracted directory name remains unchanged: apache-cloudberry-${BASE_VERSION}/ --- devops/release/cloudberry-release.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/devops/release/cloudberry-release.sh b/devops/release/cloudberry-release.sh index 3ab044d5aab..fdc4809f2f8 100755 --- a/devops/release/cloudberry-release.sh +++ b/devops/release/cloudberry-release.sh @@ -565,9 +565,10 @@ section "Staging release: $TAG" # NOTE: For RC tags like "X.Y.Z-incubating-rcN", keep the tag as-is but # generate the tarball name and top-level directory using BASE_VERSION # (without "-rcN"). This allows promoting the voted bits without rebuilding. - # Keep -rcN in the artifact filename for RC voting, but keep the extracted - # top-level directory name as BASE_VERSION (without -rcN). - TAR_NAME="apache-cloudberry-${TAG}-src.tar.gz" + # Use BASE_VERSION for both tarball filename and extracted directory name + # to align with Apache incubator release conventions. This enables direct + # 'svn mv' to release repository after voting without renaming artifacts. + TAR_NAME="apache-cloudberry-${BASE_VERSION}-src.tar.gz" TMP_DIR=$(mktemp -d) trap 'rm -rf "$TMP_DIR"' EXIT From 6e3d7d156686f2deb5fa7a66fada3e72b566a8c1 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 16:48:12 +0800 Subject: [PATCH 122/167] CI: use commit hash for Docker actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace version tags with commit hashes for Docker GitHub Actions to comply with Apache organization security requirements. Changes: - docker/setup-qemu-action@v3 → @c7c53464625b32c7a7e944ae62b3e17d2b600130 (v3.7.0) - docker/login-action@v3 → @c94ce9fb468520275223c153574b00df6fe4bcc9 (v3.7.0) - docker/setup-buildx-action@v3 → @8d2750c68a42422c14e847fe6c8ac0403b4cbd6f (v3.12.0) - docker/build-push-action@v6 → @10e90e3645eae34f1e60eeb005ba3a3d33f178e8 (v6.19.2) Affected workflows: - .github/workflows/docker-cbdb-build-containers.yml - .github/workflows/docker-cbdb-test-containers.yml Fixes https://github.com/apache/cloudberry/issues/1687 --- .github/workflows/docker-cbdb-build-containers.yml | 8 ++++---- .github/workflows/docker-cbdb-test-containers.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index dd9ea9acd27..3ef8fae00a8 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -117,13 +117,13 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 # Login to DockerHub for pushing images # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set - name: Login to Docker Hub if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -132,7 +132,7 @@ jobs: # Enable debug mode for better troubleshooting - name: Set up Docker Buildx if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: buildkitd-flags: --debug @@ -172,7 +172,7 @@ jobs: # This creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/build/${{ matrix.platform }} push: true diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 1c8e1c8a9a2..efb98d2b7a6 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -106,12 +106,12 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 # Login to DockerHub for pushing images - name: Login to Docker Hub if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -119,7 +119,7 @@ jobs: # Setup Docker Buildx for efficient multi-architecture builds - name: Set up Docker Buildx if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: buildkitd-flags: --debug @@ -142,7 +142,7 @@ jobs: # Creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/test/${{ matrix.platform }} push: true From ea46052071abd82f36f8d11a2fa249af920bf7ee Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 20 Apr 2026 18:44:48 +0800 Subject: [PATCH 123/167] CI: Add Ubuntu24.04 workflow with test matrix support This commit introduces a new GitHub Actions workflow for building and testing Apache Cloudberry on Ubuntu 24.04, enabling automated builds, DEB packaging, and regresssion testing. Triggers: - Push to main branch - Pull requests modifying this workflow file - Scheduled: Every Monday at 02:00 UTC - Manual workflow dispatch with optional test selection --- .../build-deb-cloudberry-ubuntu24.04.yml | 1892 +++++++++++++++++ 1 file changed, 1892 insertions(+) create mode 100644 .github/workflows/build-deb-cloudberry-ubuntu24.04.yml diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml new file mode 100644 index 00000000000..041eabc252b --- /dev/null +++ b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml @@ -0,0 +1,1892 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# GitHub Actions Workflow: Apache Cloudberry Build Pipeline +# -------------------------------------------------------------------- +# Description: +# +# This workflow builds, tests, and packages Apache Cloudberry on +# Ubuntu 24.04. It ensures artifact integrity and performs installation +# tests. +# +# Workflow Overview: +# 1. **Build Job**: +# - Configures and builds Apache Cloudberry. +# - Supports debug build configuration via ENABLE_DEBUG flag. +# - Runs unit tests and verifies build artifacts. +# - Creates DEB packages (regular and debug), source tarball +# and additional files for dupload utility. +# - **Key Artifacts**: DEB package, source tarball, changes and dsc files, build logs. +# +# 2. **DEB Install Test Job**: +# - Verifies DEB integrity and installs Cloudberry. +# - Validates successful installation. +# - **Key Artifacts**: Installation logs, verification results. +# +# 3. **Report Job**: +# - Aggregates job results into a final report. +# - Sends failure notifications if any step fails. +# +# Execution Environment: +# - **Runs On**: ubuntu-22.04 with ubuntu-24.04 containers. +# - **Resource Requirements**: +# - Disk: Minimum 20GB free space. +# - Memory: Minimum 8GB RAM. +# - CPU: Recommended 4+ cores. +# +# Triggers: +# - Push to `main` branch. +# - Pull request that modifies this workflow file. +# - Scheduled: Every Monday at 02:00 UTC. +# - Manual workflow dispatch. +# +# Container Images: +# - **Build**: `apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest` +# - **Test**: `apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest` +# +# Artifacts: +# - DEB Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Changes and DSC files (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). +# +# Notes: +# - Supports concurrent job execution. +# - Supports debug builds with preserved symbols. +# -------------------------------------------------------------------- + +name: Apache Cloudberry Debian Build + +on: + push: + branches: [main, REL_2_STABLE] + pull_request: + paths: + - '.github/workflows/build-deb-cloudberry-ubuntu24.04.yml' + # We can enable the PR test when needed + # branches: [main, REL_2_STABLE] + # types: [opened, synchronize, reopened, edited] + schedule: + # Run every Monday at 02:00 UTC + - cron: '0 2 * * 1' + workflow_dispatch: # Manual trigger + inputs: + test_selection: + description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' + required: false + default: 'all' + type: string + reuse_artifacts_from_run_id: + description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' + required: false + default: '' + type: string + +# Note: Step details, logs, and artifacts require users to be logged into GitHub +# even for public repositories. This is a GitHub security feature and cannot +# be overridden by permissions. + +permissions: + # READ permissions allow viewing repository contents + contents: read # Required for checking out code and reading repository files + + # READ permissions for packages (Container registry, etc) + packages: read # Allows reading from GitHub package registry + + # WRITE permissions for actions includes read access to: + # - Workflow runs + # - Artifacts (requires GitHub login) + # - Logs (requires GitHub login) + actions: write + + # READ permissions for checks API: + # - Step details visibility (requires GitHub login) + # - Check run status and details + checks: read + + # READ permissions for pull request metadata: + # - PR status + # - Associated checks + # - Review states + pull-requests: read + +env: + LOG_RETENTION_DAYS: 7 + ENABLE_DEBUG: false + +jobs: + + ## ====================================================================== + ## Job: check-skip + ## ====================================================================== + + check-skip: + runs-on: ubuntu-22.04 + outputs: + should_skip: ${{ steps.skip-check.outputs.should_skip }} + steps: + - id: skip-check + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_TITLE: ${{ github.event.pull_request.title || '' }} + PR_BODY: ${{ github.event.pull_request.body || '' }} + run: | + # Default to not skipping + echo "should_skip=false" >> "$GITHUB_OUTPUT" + + # Apply skip logic only for pull_request events + if [[ "$EVENT_NAME" == "pull_request" ]]; then + # Combine PR title and body for skip check + MESSAGE="${PR_TITLE}\n${PR_BODY}" + + # Escape special characters using printf %s + ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") + + echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" + + # Check for skip patterns + if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then + echo "should_skip=true" >> "$GITHUB_OUTPUT" + fi + else + echo "Skip logic is not applied for $EVENT_NAME events." + fi + + - name: Report Skip Status + if: steps.skip-check.outputs.should_skip == 'true' + run: | + echo "CI Skip flag detected in PR - skipping all checks." + exit 0 + + ## ====================================================================== + ## Job: prepare-test-matrix-deb + ## ====================================================================== + + prepare-test-matrix-deb: + runs-on: ubuntu-22.04 + needs: [check-skip] + if: needs.check-skip.outputs.should_skip != 'true' + outputs: + test-matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - id: set-matrix + run: | + echo "=== Matrix Preparation Diagnostics ===" + echo "Event type: ${{ github.event_name }}" + echo "Test selection input: '${{ github.event.inputs.test_selection }}'" + + # Define defaults + DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 + DEFAULT_ENABLE_CGROUPS=false + DEFAULT_ENABLE_CORE_CHECK=true + DEFAULT_PG_SETTINGS_OPTIMIZER="" + + # Define base test configurations + ALL_TESTS='{ + "include": [ + {"test":"ic-deb-good-opt-off", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"off"} + }, + {"test":"ic-deb-good-opt-on", + "make_configs":["src/test/regress:installcheck-good"], + "pg_settings":{"optimizer":"on"} + }, + {"test":"pax-ic-deb-good-opt-off", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"off", + "default_table_access_method":"pax" + } + }, + {"test":"pax-ic-deb-good-opt-on", + "make_configs":[ + "contrib/pax_storage/:pax-test", + "contrib/pax_storage/:regress_test" + ], + "pg_settings":{ + "optimizer":"on", + "default_table_access_method":"pax" + } + }, + {"test":"ic-deb-contrib", + "make_configs":["contrib/auto_explain:installcheck", + "contrib/amcheck:installcheck", + "contrib/citext:installcheck", + "contrib/btree_gin:installcheck", + "contrib/btree_gist:installcheck", + "contrib/dblink:installcheck", + "contrib/dict_int:installcheck", + "contrib/dict_xsyn:installcheck", + "contrib/extprotocol:installcheck", + "contrib/file_fdw:installcheck", + "contrib/formatter_fixedwidth:installcheck", + "contrib/hstore:installcheck", + "contrib/indexscan:installcheck", + "contrib/pg_trgm:installcheck", + "contrib/indexscan:installcheck", + "contrib/pgcrypto:installcheck", + "contrib/pgstattuple:installcheck", + "contrib/tablefunc:installcheck", + "contrib/passwordcheck:installcheck", + "contrib/pg_buffercache:installcheck", + "contrib/sslinfo:installcheck"] + }, + {"test":"ic-deb-gpcontrib", + "make_configs":["gpcontrib/orafce:installcheck", + "gpcontrib/zstd:installcheck", + "gpcontrib/gp_sparse_vector:installcheck", + "gpcontrib/gp_toolkit:installcheck"] + }, + {"test":"gpcontrib-gp-stats-collector", + "make_configs":["gpcontrib/gp_stats_collector:installcheck"], + "extension":"gp_stats_collector" + }, + {"test":"ic-cbdb-parallel", + "make_configs":["src/test/regress:installcheck-cbdb-parallel"] + } + ] + }' + + # Function to apply defaults + apply_defaults() { + echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ + --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ + --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ + --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ + 'def get_defaults: + { + num_primary_mirror_pairs: ($npm|tonumber), + enable_cgroups: $ec, + enable_core_check: $ecc, + pg_settings: { + optimizer: $opt + } + }; + get_defaults * .' + } + + # Extract all valid test names from ALL_TESTS + VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') + + # Parse input test selection + IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" + + # Default to all tests if selection is empty or 'all' + if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then + mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" + fi + + # Validate and filter selected tests + INVALID_TESTS=() + FILTERED_TESTS=() + for TEST in "${SELECTED_TESTS[@]}"; do + TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace + if echo "$VALID_TESTS" | grep -qw "$TEST"; then + FILTERED_TESTS+=("$TEST") + else + INVALID_TESTS+=("$TEST") + fi + done + + # Handle invalid tests + if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then + echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" + echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" + exit 1 + fi + + # Build result JSON with defaults applied + RESULT='{"include":[' + FIRST=true + for TEST in "${FILTERED_TESTS[@]}"; do + CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") + FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") + if [[ "$FIRST" == true ]]; then + FIRST=false + else + RESULT="${RESULT}," + fi + RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" + done + RESULT="${RESULT}]}" + + # Output the matrix for GitHub Actions + echo "Final matrix configuration:" + echo "$RESULT" | jq . + + # Fix: Use block redirection + { + echo "matrix<> "$GITHUB_OUTPUT" + + echo "=== Matrix Preparation Complete ===" + + ## ====================================================================== + ## Job: build-deb + ## ====================================================================== + + build-deb: + name: Build Apache Cloudberry DEB (Ubuntu 24.04) + env: + JOB_TYPE: build + needs: [check-skip] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + if: github.event.inputs.reuse_artifacts_from_run_id == '' + outputs: + build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Set build timestamp + id: set_timestamp # Add an ID to reference this step + run: | + timestamp=$(date +'%Y%m%d_%H%M%S') + echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs + echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable + + - name: Checkout Apache Cloudberry + uses: actions/checkout@v4 + with: + fetch-depth: 1 + submodules: true + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Generate Build Job Summary Start + run: | + { + echo "# Build Job Summary (Ubuntu 24.04)" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" + echo "- OS Version: $(lsb_release -sd)" + echo "- GCC Version: $(gcc --version | head -n1)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run Apache Cloudberry configure script + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure script failed" + exit 1 + fi + + - name: Run Apache Cloudberry build script + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build script failed" + exit 1 + fi + + - name: Verify build artifacts + shell: bash + run: | + set -eo pipefail + + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + echo "Verifying build artifacts..." + { + echo "=== Build Artifacts Verification ===" + echo "Timestamp: $(date -u)" + + if [ ! -d "${BUILD_DESTINATION}" ]; then + echo "::error::Build artifacts directory not found" + exit 1 + fi + + # Verify critical binaries + critical_binaries=( + "${BUILD_DESTINATION}/bin/postgres" + "${BUILD_DESTINATION}/bin/psql" + ) + + echo "Checking critical binaries..." + for binary in "${critical_binaries[@]}"; do + if [ ! -f "$binary" ]; then + echo "::error::Critical binary missing: $binary" + exit 1 + fi + if [ ! -x "$binary" ]; then + echo "::error::Binary not executable: $binary" + exit 1 + fi + echo "Binary verified: $binary" + ls -l "$binary" + done + + # Test binary execution + echo "Testing binary execution..." + if ! ${BUILD_DESTINATION}/bin/postgres --version; then + echo "::error::postgres binary verification failed" + exit 1 + fi + if ! ${BUILD_DESTINATION}/bin/psql --version; then + echo "::error::psql binary verification failed" + exit 1 + fi + + echo "All build artifacts verified successfully" + } 2>&1 | tee -a build-logs/details/build-verification.log + + - name: Create Source tarball, create DEB and verify artifacts + shell: bash + env: + CBDB_VERSION: 99.0.0 + BUILD_NUMBER: 1 + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Artifact Creation Log ===" + echo "Timestamp: $(date -u)" + + cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu24.04/* debian/ + chown -R "$(whoami)" debian + chmod -x debian/*install + + # replace not supported symbols in version + CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/\//./g") + CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/_/-/g") + + echo "We will built ${CBDB_VERSION}" + export BUILD_DESTINATION=${SRC_DIR}/debian/build + + if ! ${SRC_DIR}/devops/build/packaging/deb/build-deb.sh -v $CBDB_VERSION; then + echo "::error::Build script failed" + exit 1 + fi + + ARCH=$(dpkg --print-architecture) + # Detect OS distribution (e.g., ubuntu24.04, debian12) + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') + else + OS_DISTRO="unknown" + fi + CBDB_PKG_VERSION=${CBDB_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} + + echo "Produced artifacts" + ls -l ../ + + echo "Copy artifacts to subdirectory for sign/upload" + mkdir ${SRC_DIR}/deb + DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".deb + DBG_DEB_FILE="apache-cloudberry-db-incubating-dbgsym_${CBDB_PKG_VERSION}"_"${ARCH}".ddeb + CHANGES_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".changes + BUILDINFO_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".buildinfo + DSC_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".dsc + SOURCE_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".tar.xz + cp ../"${DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${DBG_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${CHANGES_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${BUILDINFO_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${DSC_DEB_FILE}" "${SRC_DIR}/deb" + cp ../"${SOURCE_FILE}" "${SRC_DIR}/deb" + mkdir "${SRC_DIR}/deb/debian" + cp debian/changelog "${SRC_DIR}/deb/debian" + + # Get package information + echo "Package Information:" + dpkg --info "${SRC_DIR}/deb/${DEB_FILE}" + dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" + + # Verify critical files in DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" | grep -c "${binary}$"; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + # Record checksums + echo "Calculating checksums..." + sha256sum "${SRC_DIR}/deb/${DEB_FILE}" | tee -a build-logs/details/checksums.log + + echo "Artifacts created and verified successfully" + + + } 2>&1 | tee -a build-logs/details/artifact-creation.log + + - name: Run Apache Cloudberry unittest script + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then + echo "::error::Unittest script failed" + exit 1 + fi + + - name: Generate Build Job Summary End + run: | + { + echo "## Build Results" + echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload build logs + uses: actions/upload-artifact@v4 + with: + name: build-logs-ubuntu24.04-${{ env.BUILD_TIMESTAMP }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Cloudberry DEB build artifacts + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + deb/*.deb + deb/*.ddeb + + - name: Upload Cloudberry deb source build artifacts + uses: actions/upload-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-source-build-artifacts + retention-days: ${{ env.LOG_RETENTION_DAYS }} + if-no-files-found: error + path: | + deb/*.tar.xz + deb/*.changes + deb/*.dsc + deb/*.buildinfo + deb/debian/changelog + + ## ====================================================================== + ## Job: deb-install-test + ## ====================================================================== + + deb-install-test: + name: DEB Install Test Apache Cloudberry (Ubuntu 24.04) + needs: [check-skip, build-deb] + if: | + !cancelled() && + (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') && + github.event.inputs.reuse_artifacts_from_run_id == '' + runs-on: ubuntu-22.04 + timeout-minutes: 120 + + container: + image: apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest + options: >- + --user root + -h cdw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "DEB install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Download Cloudberry DEB build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + path: ${{ github.workspace }}/deb_build_artifacts + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + merge-multiple: false + + - name: Cloudberry Environment Initialization + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + LOGS_DIR: install-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Verify DEB artifacts + id: verify-artifacts + shell: bash + run: | + set -eo pipefail + + DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) + if [ ! -f "${DEB_FILE}" ]; then + echo "::error::DEB file not found" + exit 1 + fi + + echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying DEB artifacts..." + { + echo "=== DEB Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + + # Get DEB metadata and verify contents + echo "Package Information:" + dpkg-deb -f "${DEB_FILE}" + + # Get key DEB attributes for verification + DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) + DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) + echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + echo "DEB Details:" + echo "- Version: ${DEB_VERSION}" + echo "- Release: ${DEB_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${DEB_FILE}" + + } 2>&1 | tee -a install-logs/details/deb-verification.log + + - name: Install Cloudberry DEB + shell: bash + env: + DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} + DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} + DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${DEB_FILE}" ]; then + echo "::error::DEB_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== DEB Installation Log ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + echo "Version: ${DEB_VERSION}" + echo "Release: ${DEB_RELEASE}" + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install DEB + echo "Starting installation..." + apt-get update + if ! apt-get -y install "${DEB_FILE}"; then + echo "::error::DEB installation failed" + exit 1 + fi + + # Change ownership back to gpadmin - it is needed for future tests + chown -R gpadmin:gpadmin /usr/local/cloudberry-db + + echo "Installation completed successfully" + dpkg-query -s apache-cloudberry-db-incubating + echo "Installed files:" + dpkg-query -L apache-cloudberry-db-incubating + } 2>&1 | tee -a install-logs/details/deb-installation.log + + - name: Upload install logs + uses: actions/upload-artifact@v4 + with: + name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + install-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Generate Install Test Job Summary End + if: always() + shell: bash {0} + run: | + { + echo "# Installed Package Summary (Ubuntu 24.04)" + echo "\`\`\`" + + dpkg-query -s apache-cloudberry-db-incubating + echo "\`\`\`" + } >> "$GITHUB_STEP_SUMMARY" || true + + ## ====================================================================== + ## Job: test-deb + ## ====================================================================== + + test-deb: + name: ${{ matrix.test }} (Ubuntu 24.04) + needs: [check-skip, build-deb, prepare-test-matrix-deb] + if: | + !cancelled() && + (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') + runs-on: ubuntu-22.04 + timeout-minutes: 120 + # actionlint-allow matrix[*].pg_settings + strategy: + fail-fast: false # Continue with other tests if one fails + matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} + + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest + options: >- + --privileged + --user root + --hostname cdw + --shm-size=2gb + --ulimit core=-1 + --cgroupns=host + -v /sys/fs/cgroup:/sys/fs/cgroup:rw + -v /usr/share:/host_usr_share + -v /usr/local:/host_usr_local + -v /opt:/host_opt + + steps: + - name: Free Disk Space + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "=== Disk space before cleanup ===" + df -h / + + # Remove pre-installed tools from host to free disk space + rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache + rm -rf /host_usr_local/lib/android || true # Android SDK + rm -rf /host_usr_share/dotnet || true # .NET SDK + rm -rf /host_opt/ghc || true # Haskell GHC + rm -rf /host_usr_local/.ghcup || true # Haskell GHCup + rm -rf /host_usr_share/swift || true # Swift + rm -rf /host_usr_local/share/powershell || true # PowerShell + rm -rf /host_usr_local/share/chromium || true # Chromium + rm -rf /host_usr_share/miniconda || true # Miniconda + rm -rf /host_opt/az || true # Azure CLI + rm -rf /host_usr_share/sbt || true # Scala Build Tool + + echo "=== Disk space after cleanup ===" + df -h / + + - name: Skip Check + if: needs.check-skip.outputs.should_skip == 'true' + run: | + echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" + exit 0 + + - name: Use timestamp from previous job + if: needs.check-skip.outputs.should_skip != 'true' + run: | + echo "Timestamp from output: ${{ needs.build-deb.outputs.build_timestamp }}" + + - name: Cloudberry Environment Initialization + shell: bash + env: + LOGS_DIR: build-logs + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + + mkdir -p "${LOGS_DIR}/details" + chown -R gpadmin:gpadmin . + chmod -R 755 . + chmod 777 "${LOGS_DIR}" + + df -kh / + rm -rf /__t/* + df -kh / + + df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" + free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" + + { + echo "=== Environment Information ===" + uname -a + df -h + free -h + env + } | tee -a "${LOGS_DIR}/details/environment.log" + + echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" + + - name: Setup cgroups + if: needs.check-skip.outputs.should_skip != 'true' + shell: bash + run: | + set -uxo pipefail + + if [ "${{ matrix.enable_cgroups }}" = "true" ]; then + + echo "Current mounts:" + mount | grep cgroup + + CGROUP_BASEDIR=/sys/fs/cgroup + + # 1. Basic setup with permissions + sudo chmod -R 777 ${CGROUP_BASEDIR}/ + sudo mkdir -p ${CGROUP_BASEDIR}/gpdb + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 2. Enable controllers + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true + sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true + + # 3. CPU settings + sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true + sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true + sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true + + # 4. Memory settings + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true + sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true + sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true + + # 5. IO settings + echo "Available block devices:" + lsblk + + sudo bash -c " + if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then + echo 'Detected IO devices:' + cat \${CGROUP_BASEDIR}/gpdb/io.stat + fi + echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true + " + + # 6. Fix permissions again after all writes + sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb + sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb + + # 7. Check required files + echo "Checking required files:" + required_files=( + "cgroup.procs" + "cpu.max" + "cpu.pressure" + "cpu.weight" + "cpu.weight.nice" + "cpu.stat" + "cpuset.cpus" + "cpuset.mems" + "cpuset.cpus.effective" + "cpuset.mems.effective" + "memory.current" + "io.max" + ) + + for file in "${required_files[@]}"; do + if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then + echo "✓ $file exists" + ls -l "${CGROUP_BASEDIR}/gpdb/$file" + else + echo "✗ $file missing" + fi + done + + # 8. Test subdirectory creation + echo "Testing subdirectory creation..." + sudo -u gpadmin bash -c " + TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 + if mkdir -p \$TEST_DIR; then + echo 'Created test directory' + sudo chmod -R 777 \$TEST_DIR + if echo \$\$ > \$TEST_DIR/cgroup.procs; then + echo 'Successfully wrote to cgroup.procs' + cat \$TEST_DIR/cgroup.procs + # Move processes back to parent before cleanup + echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs + else + echo 'Failed to write to cgroup.procs' + ls -la \$TEST_DIR/cgroup.procs + fi + ls -la \$TEST_DIR/ + rmdir \$TEST_DIR || { + echo 'Moving all processes to parent before cleanup' + cat \$TEST_DIR/cgroup.procs | while read pid; do + echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true + done + rmdir \$TEST_DIR + } + else + echo 'Failed to create test directory' + fi + " + + # 9. Verify setup as gpadmin user + echo "Testing cgroup access as gpadmin..." + sudo -u gpadmin bash -c " + echo 'Checking mounts...' + mount | grep cgroup + + echo 'Checking /proc/self/mounts...' + cat /proc/self/mounts | grep cgroup + + if ! grep -q cgroup2 /proc/self/mounts; then + echo 'ERROR: cgroup2 mount NOT visible to gpadmin' + exit 1 + fi + echo 'SUCCESS: cgroup2 mount visible to gpadmin' + + if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then + echo 'ERROR: gpadmin cannot write to gpdb cgroup' + exit 1 + fi + echo 'SUCCESS: gpadmin can write to gpdb cgroup' + + echo 'Verifying key files content:' + echo 'cpu.max:' + cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' + echo 'cpuset.cpus:' + cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' + echo 'cgroup.subtree_control:' + cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' + " + + # 10. Show final state + echo "Final cgroup state:" + ls -la ${CGROUP_BASEDIR}/gpdb/ + echo "Cgroup setup completed successfully" + else + echo "Cgroup setup skipped" + fi + + - name: "Generate Test Job Summary Start: ${{ matrix.test }}" + if: always() + run: | + { + echo "# Test Job Summary: ${{ matrix.test }} (Ubuntu 24.04)" + echo "## Environment" + echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## Skip Status" + echo "✓ Test execution skipped via CI skip flag" + else + echo "- OS Version: $(cat /etc/redhat-release)" + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Download Cloudberry DEB build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts + path: ${{ github.workspace }}/deb_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download Cloudberry Source build artifacts + if: needs.check-skip.outputs.should_skip != 'true' + uses: actions/download-artifact@v4 + with: + name: apache-cloudberry-db-incubating-deb-source-build-artifacts + path: ${{ github.workspace }}/source_build_artifacts + merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Verify DEB artifacts + if: needs.check-skip.outputs.should_skip != 'true' + id: verify-artifacts + shell: bash + run: | + set -eo pipefail + + SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-db-incubating_*.tar.xz) + if [ ! -f "${SRC_TARBALL_FILE}" ]; then + echo "::error::SRC TARBALL file not found" + exit 1 + fi + + echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying SRC TARBALL artifacts..." + { + echo "=== SRC TARBALL Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${SRC_TARBALL_FILE}" + + } 2>&1 | tee -a build-logs/details/src-tarball-verification.log + + DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) + if [ ! -f "${DEB_FILE}" ]; then + echo "::error::DEB file not found" + exit 1 + fi + + echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" + + echo "Verifying DEB artifacts..." + { + echo "=== DEB Verification Summary ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + + # Get DEB metadata and verify contents + echo "Package Information:" + dpkg-deb -f "${DEB_FILE}" + + # Get key DEB attributes for verification + DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) + DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) + echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" + + # Verify expected binaries are in the DEB + echo "Verifying critical files in DEB..." + for binary in "bin/postgres" "bin/psql"; do + if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then + echo "::error::Critical binary '${binary}' not found in DEB" + exit 1 + fi + done + + echo "DEB Details:" + echo "- Version: ${DEB_VERSION}" + echo "- Release: ${DEB_RELEASE}" + + # Calculate and store checksum + echo "Checksum:" + sha256sum "${DEB_FILE}" + + } 2>&1 | tee -a build-logs/details/deb-verification.log + + - name: Install Cloudberry DEB + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} + DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} + DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} + run: | + set -eo pipefail + + if [ -z "${DEB_FILE}" ]; then + echo "::error::DEB_FILE environment variable is not set" + exit 1 + fi + + { + echo "=== DEB Installation Log ===" + echo "Timestamp: $(date -u)" + echo "DEB File: ${DEB_FILE}" + echo "Version: ${DEB_VERSION}" + echo "Release: ${DEB_RELEASE}" + + # Clean install location + rm -rf /usr/local/cloudberry-db + + # Install DEB + echo "Starting installation..." + apt-get update + if ! apt-get -y install "${DEB_FILE}"; then + echo "::error::DEB installation failed" + exit 1 + fi + + # Change ownership back to gpadmin - it is needed for future tests + chown -R gpadmin:gpadmin /usr/local/cloudberry-db + + echo "Installation completed successfully" + dpkg-query -s apache-cloudberry-db-incubating + echo "Installed files:" + dpkg-query -L apache-cloudberry-db-incubating + } 2>&1 | tee -a build-logs/details/deb-installation.log + + - name: Extract source tarball + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + echo "=== Source Extraction Log ===" + echo "Timestamp: $(date -u)" + + echo "Starting extraction..." + file "${SRC_TARBALL_FILE}" + if ! time tar xf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then + echo "::error::Source extraction failed" + exit 1 + fi + + echo "Extraction completed successfully" + echo "Extracted contents:" + ls -la "${SRC_DIR}/../cloudberry" + echo "Directory size:" + du -sh "${SRC_DIR}/../cloudberry" + } 2>&1 | tee -a build-logs/details/source-extraction.log + + - name: Prepare DEB Environment + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + + # change ownership to gpadmin + chown -R gpadmin "${SRC_DIR}/../cloudberry" + touch build-logs/sections.log + chown gpadmin build-logs/sections.log + chmod 777 build-logs + + # configure link lib directory to temporary location, fix it + rm -rf "${SRC_DIR}"/debian/build/lib + ln -sf /usr/cloudberry-db/lib "${SRC_DIR}"/debian/build/lib + + # check if regress.so exists in src directory - it is needed for contrib/dblink tests + if [ ! -f ${SRC_DIR}/src/test/regress/regress.so ]; then + ln -sf /usr/cloudberry-db/lib/postgresql/regress.so ${SRC_DIR}/src/test/regress/regress.so + fi + + # FIXME + # temporary install gdb - delete after creating new docker build/test contaners + apt-get update + apt-get -y install gdb + + } 2>&1 | tee -a build-logs/details/prepare-deb-env.log + + - name: Create Apache Cloudberry demo cluster + if: success() && needs.check-skip.outputs.should_skip != 'true' + shell: bash + env: + SRC_DIR: ${{ github.workspace }} + run: | + set -eo pipefail + + { + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + + # Build BLDWRAP_POSTGRES_CONF_ADDONS for shared_preload_libraries if specified + EXTRA_CONF="" + if [[ -n "${{ matrix.shared_preload_libraries }}" ]]; then + EXTRA_CONF="shared_preload_libraries='${{ matrix.shared_preload_libraries }}'" + echo "Adding shared_preload_libraries: ${{ matrix.shared_preload_libraries }}" + fi + + if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' BLDWRAP_POSTGRES_CONF_ADDONS=\"${EXTRA_CONF}\" SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + + } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log + + - name: "Run Tests: ${{ matrix.test }}" + if: success() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + BUILD_DESTINATION: ${{ github.workspace }}/debian/build + shell: bash {0} + run: | + set -o pipefail + + # Initialize test status + overall_status=0 + + # Create logs directory structure + mkdir -p build-logs/details + + # Core file config + mkdir -p "/tmp/cloudberry-cores" + chmod 1777 "/tmp/cloudberry-cores" + sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" + sysctl kernel.core_pattern + su - gpadmin -c "ulimit -c" + + # WARNING: PostgreSQL Settings + # When adding new pg_settings key/value pairs: + # 1. Add a new check below for the setting + # 2. Follow the same pattern as optimizer + # 3. Update matrix entries to include the new setting + + + # Create extension if required + if [[ "${{ matrix.extension != '' }}" == "true" ]]; then + case "${{ matrix.extension }}" in + gp_stats_collector) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_stats_collector extension" + exit 1 + fi + ;; + *) + echo "Unknown extension: ${{ matrix.extension }}" + exit 1 + ;; + esac + fi + + # Set PostgreSQL options if defined + PG_OPTS="" + if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" + fi + + if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then + PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" + fi + + # Read configs into array + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting test execution for ${{ matrix.test }} ===" + echo "Number of configurations to execute: ${#configs[@]}" + echo "" + + # Execute each config separately + for ((i=0; i<${#configs[@]}; i++)); do + config="${configs[$i]}" + IFS=':' read -r dir target <<< "$config" + + echo "=== Executing configuration $((i+1))/${#configs[@]} ===" + echo "Make command: make -C $dir $target" + echo "Environment:" + echo "- PGOPTIONS: ${PG_OPTS}" + + # Create unique log file for this configuration + config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" + + # Clean up any existing core files + echo "Cleaning up existing core files..." + rm -f /tmp/cloudberry-cores/core-* + + # Execute test script with proper environment setup + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + MAKE_NAME='${{ matrix.test }}-config$i' \ + MAKE_TARGET='$target' \ + MAKE_DIRECTORY='-C $dir' \ + PGOPTIONS='${PG_OPTS}' \ + SRC_DIR='${SRC_DIR}' \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ + 2>&1 | tee "$config_log"; then + echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" + overall_status=1 + fi + + # Check for results directory + results_dir="${dir}/results" + + if [[ -d "$results_dir" ]]; then + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + + find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log + else + echo "-----------------------------------------" + echo "Results directory $results_dir does not exit" + echo "-----------------------------------------" + fi + + # Analyze any core files generated by this test configuration + echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." + test_id="${{ matrix.test }}-config$i" + + # List the cores directory + echo "-----------------------------------------" + echo "Cores directory: /tmp/cloudberry-cores" + echo "Contents of cores directory:" + ls -Rl "/tmp/cloudberry-cores" + echo "-----------------------------------------" + + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" + core_analysis_rc=$? + case "$core_analysis_rc" in + 0) echo "No core dumps found for this configuration" ;; + 1) echo "Core dumps were found and analyzed successfully" ;; + 2) echo "::warning::Issues encountered during core dump analysis" ;; + *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; + esac + + echo "Log file: $config_log" + echo "=== End configuration $((i+1)) execution ===" + echo "" + done + + echo "=== Test execution completed ===" + echo "Log files:" + ls -l build-logs/details/ + + # Store number of configurations for parsing step + echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" + + # Report overall status + if [ $overall_status -eq 0 ]; then + echo "All test executions completed successfully" + else + echo "::warning::Some test executions failed, check individual logs for details" + fi + + exit $overall_status + + - name: "Parse Test Results: ${{ matrix.test }}" + id: test-results + if: always() && needs.check-skip.outputs.should_skip != 'true' + env: + SRC_DIR: ${{ github.workspace }} + shell: bash {0} + run: | + set -o pipefail + + overall_status=0 + + # Get configs array to create context for results + IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" + + echo "=== Starting results parsing for ${{ matrix.test }} ===" + echo "Number of configurations to parse: ${#configs[@]}" + echo "" + + # Parse each configuration's results independently + for ((i=0; i "test_results.$i.txt" + overall_status=1 + continue + fi + + # Parse this configuration's results + + MAKE_NAME="${{ matrix.test }}-config$i" \ + "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" + status_code=$? + + { + echo "SUITE_NAME=${{ matrix.test }}" + echo "DIR=${dir}" + echo "TARGET=${target}" + } >> test_results.txt + + # Process return code + case $status_code in + 0) # All tests passed + echo "All tests passed successfully" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + ;; + 1) # Tests failed but parsed successfully + echo "Test failures detected but properly parsed" + if [ -f test_results.txt ]; then + (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" + rm test_results.txt + fi + overall_status=1 + ;; + 2) # Parse error or missing file + echo "::warning::Could not parse test results properly for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=parse_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + *) # Unexpected error + echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" + { + echo "MAKE_COMMAND=\"make -C $dir $target\"" + echo "STATUS=unknown_error" + echo "TOTAL_TESTS=0" + echo "FAILED_TESTS=0" + echo "PASSED_TESTS=0" + echo "IGNORED_TESTS=0" + } | tee "test_results.${{ matrix.test }}.$i.txt" + overall_status=1 + ;; + esac + + echo "Results stored in test_results.$i.txt" + echo "=== End parsing for configuration $((i+1)) ===" + echo "" + done + + # Report status of results files + echo "=== Results file status ===" + echo "Generated results files:" + for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + build-logs/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload Test Metadata + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-metadata-${{ matrix.test }} + path: | + test_results*.txt + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test results files + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + - name: Upload test regression logs + if: failure() || cancelled() + uses: actions/upload-artifact@v4 + with: + name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + path: | + **/regression.out + **/regression.diffs + **/results/ + gpAux/gpdemo/datadirs/standby/log/ + gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ + gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ + gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ + gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ + gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ + retention-days: ${{ env.LOG_RETENTION_DAYS }} + + ## ====================================================================== + ## Job: report-deb + ## ====================================================================== + + report-deb: + name: Generate Apache Cloudberry Build Report (Ubuntu 24.04) + needs: [check-skip, build-deb, prepare-test-matrix-deb, deb-install-test, test-deb] + if: always() + runs-on: ubuntu-22.04 + steps: + - name: Generate Final Report + run: | + { + echo "# Apache Cloudberry Build Pipeline Report" + + if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then + echo "## CI Skip Status" + echo "✅ CI checks skipped via skip flag" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + else + echo "## Job Status" + echo "- Build Job: ${{ needs.build-deb.result }}" + echo "- Test Job: ${{ needs.test-deb.result }}" + echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + if [[ "${{ needs.build-deb.result }}" == "success" && "${{ needs.test-deb.result }}" == "success" ]]; then + echo "✅ Pipeline completed successfully" + else + echo "⚠️ Pipeline completed with failures" + + if [[ "${{ needs.build-deb.result }}" != "success" ]]; then + echo "### Build Job Failure" + echo "Check build logs for details" + fi + + if [[ "${{ needs.test-deb.result }}" != "success" ]]; then + echo "### Test Job Failure" + echo "Check test logs and regression files for details" + fi + fi + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Notify on failure + if: | + needs.check-skip.outputs.should_skip != 'true' && + (needs.build-deb.result != 'success' || needs.test-deb.result != 'success') + run: | + echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" + echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + echo "Build Result: ${{ needs.build-deb.result }}" + echo "Test Result: ${{ needs.test-deb.result }}" From 4ac49853ebb47e3f5a95b1027e1f9f0efcd3df49 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Tue, 21 Apr 2026 10:41:47 +0800 Subject: [PATCH 124/167] Fix Python 3.12 SyntaxWarning in orphaned_toast_tables_check.py Use raw string literal (r""") for SQL query in orphaned_toast_tables_check.py to avoid SyntaxWarning on Python 3.12. The query contains `\d` for PostgreSQL regex which Python 3.12 incorrectly interprets as an invalid escape sequence, causing test failures on Ubuntu 24.04. See: https://github.com/apache/cloudberry/issues/1686 --- gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py index 21ec8d18047..789e1b139d2 100644 --- a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py @@ -25,7 +25,7 @@ def __init__(self): # pg_depend back to pg_class, and if the table oids don't match and/or # one is missing, the TOAST table is considered to be an orphan. # Note: Handles toast tables which is created/used by InitTempTableNamespace(). - self.orphaned_toast_tables_query = """ + self.orphaned_toast_tables_query = r""" SELECT gp_segment_id AS content_id, toast_table_oid, From bf00cea87314d63bbdf2006b5fee5d75ec8cc7cc Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Wed, 22 Apr 2026 20:38:59 -0700 Subject: [PATCH 125/167] Fix colNDVBySeg attnum index mismatch in column-specific ANALYZE (#1680) * Fix colNDVBySeg index mismatch in do_analyze_rel When ANALYZE is run on specific columns (e.g., ANALYZE t (col)) or when a table has dropped columns, the vacattrstats loop index `i` diverges from the attribute's actual attnum-1 index used by colNDVBySeg. Two fixes: 1. QD side (line 887): read colNDVBySeg[attnum-1] instead of colNDVBySeg[i] when storing stadistinctbyseg. 2. Segment side (line 1011): write ctx->stadistincts[attnum-1] instead of ctx->stadistincts[i] when collecting per-segment NDV. * Add regression test for colNDVBySeg index mismatch in do_analyze_rel ANALYZE t(b) puts column b at loop index i=0 on the QD, but b has attnum=2, so attnum-1=1 != i=0. The fix in do_analyze_rel (using attnum-1 instead of i to index colNDVBySeg) ensures stadistinctbyseg is read from the correct per-segment NDV slot. Test verifies stadistinctbyseg for column b equals 100 (all distinct) rather than ~5 (NDV of column a at index 0). --- src/backend/commands/analyze.c | 4 ++-- src/test/regress/expected/analyze.out | 27 +++++++++++++++++++++++++++ src/test/regress/sql/analyze.sql | 23 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index c15dcdc8213..6f8b5e3d0ec 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -884,7 +884,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, if (Gp_role == GP_ROLE_DISPATCH && GpPolicyIsPartitioned(onerel->rd_cdbpolicy)) { - stats->stadistinctbyseg = colNDVBySeg[i]; + stats->stadistinctbyseg = colNDVBySeg[stats->attr->attnum - 1]; } stats->tupDesc = onerel->rd_att; @@ -1008,7 +1008,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, if (Gp_role == GP_ROLE_EXECUTE) { Assert(ctx->stadistincts); - ctx->stadistincts[i] = Float8GetDatum(stats->stadistinct); + ctx->stadistincts[stats->attr->attnum - 1] = Float8GetDatum(stats->stadistinct); } MemoryContextResetAndDeleteChildren(col_context); diff --git a/src/test/regress/expected/analyze.out b/src/test/regress/expected/analyze.out index 843a728c9b6..74169660301 100644 --- a/src/test/regress/expected/analyze.out +++ b/src/test/regress/expected/analyze.out @@ -1314,3 +1314,30 @@ select * from pg_stats where tablename like 'part2'; (1 row) drop table multipart cascade; +-- +-- Test column-specific ANALYZE correctly uses attnum-based NDV index (not loop index). +-- When ANALYZE t(b) is run, the QD loop has i=0 for column b (attnum=2), +-- so attnum-1=1 != i=0. Without the fix, colNDVBySeg[i=0] reads column a's NDV +-- instead of column b's NDV. +-- +CREATE TABLE analyze_col_ndv_drop (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO analyze_col_ndv_drop SELECT i%5, i, i%50 FROM generate_series(1, 100) i; +-- ANALYZE specific column b: QD loop has i=0, b.attnum=2, so attnum-1=1 != i=0 +ANALYZE analyze_col_ndv_drop (b); +-- stadistinctbyseg for b should be 100 (all distinct), not ~5 (NDV of column a at index 0) +SELECT a.attname, + CASE WHEN s.stakind1 = 8 THEN array_to_string(s.stavalues1, ',') + WHEN s.stakind2 = 8 THEN array_to_string(s.stavalues2, ',') + WHEN s.stakind3 = 8 THEN array_to_string(s.stavalues3, ',') + WHEN s.stakind4 = 8 THEN array_to_string(s.stavalues4, ',') + WHEN s.stakind5 = 8 THEN array_to_string(s.stavalues5, ',') + END AS stadistinctbyseg +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'analyze_col_ndv_drop'::regclass AND a.attname = 'b'; + attname | stadistinctbyseg +---------+------------------ + b | 100 +(1 row) + +DROP TABLE analyze_col_ndv_drop; diff --git a/src/test/regress/sql/analyze.sql b/src/test/regress/sql/analyze.sql index fa2ce8b834f..6d1c7ddd159 100644 --- a/src/test/regress/sql/analyze.sql +++ b/src/test/regress/sql/analyze.sql @@ -677,3 +677,26 @@ analyze verbose p2; select * from pg_stats where tablename like 'part2'; drop table multipart cascade; + +-- +-- Test column-specific ANALYZE correctly uses attnum-based NDV index (not loop index). +-- When ANALYZE t(b) is run, the QD loop has i=0 for column b (attnum=2), +-- so attnum-1=1 != i=0. Without the fix, colNDVBySeg[i=0] reads column a's NDV +-- instead of column b's NDV. +-- +CREATE TABLE analyze_col_ndv_drop (a int, b int, c int) DISTRIBUTED BY (a); +INSERT INTO analyze_col_ndv_drop SELECT i%5, i, i%50 FROM generate_series(1, 100) i; +-- ANALYZE specific column b: QD loop has i=0, b.attnum=2, so attnum-1=1 != i=0 +ANALYZE analyze_col_ndv_drop (b); +-- stadistinctbyseg for b should be 100 (all distinct), not ~5 (NDV of column a at index 0) +SELECT a.attname, + CASE WHEN s.stakind1 = 8 THEN array_to_string(s.stavalues1, ',') + WHEN s.stakind2 = 8 THEN array_to_string(s.stavalues2, ',') + WHEN s.stakind3 = 8 THEN array_to_string(s.stavalues3, ',') + WHEN s.stakind4 = 8 THEN array_to_string(s.stavalues4, ',') + WHEN s.stakind5 = 8 THEN array_to_string(s.stavalues5, ',') + END AS stadistinctbyseg +FROM pg_statistic s +JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum +WHERE s.starelid = 'analyze_col_ndv_drop'::regclass AND a.attname = 'b'; +DROP TABLE analyze_col_ndv_drop; From b3a973415d3f418342aca4b456a5231779f34d3c Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 11:26:16 +0800 Subject: [PATCH 126/167] DevOps: upgrade Go to 1.24.13 in Docker build images Update Go installation in all Docker build containers to use the latest Go 1.24.13 release instead of 1.23.4, with corresponding SHA256 checksums for both amd64 and arm64 architectures. Affected files: - devops/deploy/docker/build/rocky8/Dockerfile - devops/deploy/docker/build/rocky9/Dockerfile - devops/deploy/docker/build/ubuntu22.04/Dockerfile - devops/deploy/docker/build/ubuntu24.04/Dockerfile Updated SHA256 checksums: - linux-amd64: 1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730 - linux-arm64: 74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3 --- devops/deploy/docker/build/rocky8/Dockerfile | 6 +++--- devops/deploy/docker/build/rocky9/Dockerfile | 6 +++--- devops/deploy/docker/build/ubuntu22.04/Dockerfile | 6 +++--- devops/deploy/docker/build/ubuntu24.04/Dockerfile | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/devops/deploy/docker/build/rocky8/Dockerfile b/devops/deploy/docker/build/rocky8/Dockerfile index 45d6706e593..2c19236bd76 100644 --- a/devops/deploy/docker/build/rocky8/Dockerfile +++ b/devops/deploy/docker/build/rocky8/Dockerfile @@ -150,14 +150,14 @@ RUN dnf makecache && \ make -j$(nproc) && \ make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ rm -rf ~/xerces-c* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/rocky9/Dockerfile b/devops/deploy/docker/build/rocky9/Dockerfile index 26190109ef0..62289f51371 100644 --- a/devops/deploy/docker/build/rocky9/Dockerfile +++ b/devops/deploy/docker/build/rocky9/Dockerfile @@ -151,14 +151,14 @@ RUN dnf makecache && \ make -j$(nproc) && \ make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ rm -rf ~/xerces-c* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/ubuntu22.04/Dockerfile b/devops/deploy/docker/build/ubuntu22.04/Dockerfile index 3023a9fce67..8c0e4cf3bac 100644 --- a/devops/deploy/docker/build/ubuntu22.04/Dockerfile +++ b/devops/deploy/docker/build/ubuntu22.04/Dockerfile @@ -144,14 +144,14 @@ RUN apt-get update && \ quilt \ unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ diff --git a/devops/deploy/docker/build/ubuntu24.04/Dockerfile b/devops/deploy/docker/build/ubuntu24.04/Dockerfile index c4f4e646720..762456d0b84 100644 --- a/devops/deploy/docker/build/ubuntu24.04/Dockerfile +++ b/devops/deploy/docker/build/ubuntu24.04/Dockerfile @@ -144,14 +144,14 @@ RUN apt-get update && \ quilt \ unzip && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - cd && GO_VERSION="go1.23.4" && \ + cd && GO_VERSION="go1.24.13" && \ ARCH=$(uname -m) && \ if [ "${ARCH}" = "aarch64" ]; then \ GO_ARCH="arm64" && \ - GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ elif [ "${ARCH}" = "x86_64" ]; then \ GO_ARCH="amd64" && \ - GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ else \ echo "Unsupported architecture: ${ARCH}" && exit 1; \ fi && \ From 2cd27adfb5c14e5f1664a510271a2730257ff8fd Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 14:24:20 +0800 Subject: [PATCH 127/167] CI: update docker/setup-qemu-action to v4.0.0 See: https://github.com/apache/infrastructure-actions/blob/2b6ec5f38ac73c7c5970f3b4f863e8d15bf12d7d/actions.yml#L326 --- .github/workflows/docker-cbdb-build-containers.yml | 2 +- .github/workflows/docker-cbdb-test-containers.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 3ef8fae00a8..d42139d0fae 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -117,7 +117,7 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Login to DockerHub for pushing images # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index efb98d2b7a6..36d320f0737 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -106,7 +106,7 @@ jobs: # This allows building ARM64 images on AMD64 infrastructure and vice versa - name: Set up QEMU if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' }} - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Login to DockerHub for pushing images - name: Login to Docker Hub From 859dd5bdc6d160ef4235a69fede6040a49bb9b17 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 22 Apr 2026 17:27:13 +0800 Subject: [PATCH 128/167] Sandbox: add Cloudberry 2.1.0 release support Main changes: - Update default CODEBASE_VERSION from 2.0.0 to 2.1.0 in .env - Update documentation examples to use version 2.1.0 in README.md - Update help message example version in run.sh - Switch to Apache mirror system for downloading release tarball using closer.lua for better download reliability and speed - Replace wget with curl for source download in Dockerfile This change ensures the sandbox environment defaults to the latest Apache Cloudberry 2.1.0 release and uses the recommended Apache mirror download method. --- devops/sandbox/.env | 2 +- devops/sandbox/Dockerfile.RELEASE.rockylinux9 | 52 ++++--------------- devops/sandbox/README.md | 4 +- devops/sandbox/run.sh | 2 +- 4 files changed, 13 insertions(+), 47 deletions(-) diff --git a/devops/sandbox/.env b/devops/sandbox/.env index 233d7c5b1b5..1ceec2e5fb7 100644 --- a/devops/sandbox/.env +++ b/devops/sandbox/.env @@ -17,5 +17,5 @@ # permissions and limitations under the License. # # -------------------------------------------------------------------- -CODEBASE_VERSION=2.0.0 +CODEBASE_VERSION=2.1.0 OS_VERSION=rockylinux9 diff --git a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 index ac394c6cb60..215c32f452d 100644 --- a/devops/sandbox/Dockerfile.RELEASE.rockylinux9 +++ b/devops/sandbox/Dockerfile.RELEASE.rockylinux9 @@ -94,6 +94,7 @@ RUN dnf makecache && \ readline-devel \ zlib-devel && \ dnf install -y --enablerepo=crb \ + liburing-devel \ libuv-devel \ libyaml-devel \ perl-IPC-Run \ @@ -120,10 +121,12 @@ USER gpadmin WORKDIR /home/gpadmin # Release version to build (Apache official tarball) -ARG CB_RELEASE_VERSION=2.0.0-incubating +ARG CB_RELEASE_VERSION=2.1.0-incubating # Download and extract the specified release version from Apache -RUN wget -nv "https://downloads.apache.org/incubator/cloudberry/${CB_RELEASE_VERSION}/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz" -O /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz && \ +# Using Apache mirror system for better download reliability and speed +RUN curl -L -o /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz \ + "https://www.apache.org/dyn/closer.lua/incubator/cloudberry/${CB_RELEASE_VERSION}/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz?action=download" && \ tar -xzf /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz -C /home/gpadmin && \ rm -f /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION}-src.tar.gz && \ mv /home/gpadmin/apache-cloudberry-${CB_RELEASE_VERSION} /home/gpadmin/cloudberry @@ -131,47 +134,9 @@ RUN wget -nv "https://downloads.apache.org/incubator/cloudberry/${CB_RELEASE_VER # Build Cloudberry using the official build scripts RUN cd /home/gpadmin/cloudberry && \ export SRC_DIR=/home/gpadmin/cloudberry && \ - mkdir -p "${SRC_DIR}/build-logs" && \ - # Ensure Cloudberry lib dir exists and has Xerces libs available - sudo rm -rf /usr/local/cloudberry-db && \ - sudo mkdir -p /usr/local/cloudberry-db/lib && \ - sudo cp -v /usr/local/xerces-c/lib/libxerces-c.so \ - /usr/local/xerces-c/lib/libxerces-c-3.*.so \ - /usr/local/cloudberry-db/lib/ && \ - sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db && \ - # Configure with required features and paths - export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:$LD_LIBRARY_PATH && \ - ./configure --prefix=/usr/local/cloudberry-db \ - --disable-external-fts \ - --enable-debug \ - --enable-cassert \ - --enable-debug-extensions \ - --enable-gpcloud \ - --enable-ic-proxy \ - --enable-mapreduce \ - --enable-orafce \ - --enable-orca \ - --enable-pax \ - --disable-pxf \ - --enable-tap-tests \ - --with-gssapi \ - --with-ldap \ - --with-libxml \ - --with-lz4 \ - --with-pam \ - --with-perl \ - --with-pgport=5432 \ - --with-python \ - --with-pythonsrc-ext \ - --with-ssl=openssl \ - --with-uuid=e2fs \ - --with-includes=/usr/local/xerces-c/include \ - --with-libraries=/usr/local/cloudberry-db/lib && \ - # Build and install - make -j$(nproc) --directory ${SRC_DIR} && \ - make -j$(nproc) --directory ${SRC_DIR}/contrib && \ - make install --directory ${SRC_DIR} && \ - make install --directory "${SRC_DIR}/contrib" + mkdir -p ${SRC_DIR}/build-logs && \ + ./devops/build/automation/cloudberry/scripts/configure-cloudberry.sh && \ + ./devops/build/automation/cloudberry/scripts/build-cloudberry.sh # -------------------------------------------------------------------- # Runtime stage: Rocky Linux 9 runtime with required dependencies @@ -192,6 +157,7 @@ RUN dnf -y update && \ krb5-libs \ libevent \ libicu \ + liburing \ libuuid \ libxml2 \ libyaml \ diff --git a/devops/sandbox/README.md b/devops/sandbox/README.md index 9f475977835..fb6a5ef80c3 100644 --- a/devops/sandbox/README.md +++ b/devops/sandbox/README.md @@ -92,14 +92,14 @@ Build and deploy steps: ```shell cd cloudberry/devops/sandbox - ./run.sh -c 2.0.0 + ./run.sh -c 2.1.0 ``` - For latest Apache Cloudberry release running across multiple containers ```shell cd cloudberry/devops/sandbox - ./run.sh -c 2.0.0 -m + ./run.sh -c 2.1.0 -m ``` - For latest main branch running on a single container diff --git a/devops/sandbox/run.sh b/devops/sandbox/run.sh index 7c266b8f64c..705442d98e1 100755 --- a/devops/sandbox/run.sh +++ b/devops/sandbox/run.sh @@ -38,7 +38,7 @@ PIP_INDEX_URL_VAR="${PIP_INDEX_URL_VAR:-$DEFAULT_PIP_INDEX_URL_VAR}" # Function to display help message function usage() { echo "Usage: $0 [-o ] [-c ] [-b] [-m]" - echo " -c Codebase version (valid values: main, local, or other available version like 2.0.0)" + echo " -c Codebase version (valid values: main, local, or other available version like 2.1.0)" echo " -t Timezone (default: America/Los_Angeles, or set via TIMEZONE_VAR environment variable)" echo " -p Python Package Index (PyPI) (default: https://pypi.org/simple, or set via PIP_INDEX_URL_VAR environment variable)" echo " -b Build only, do not run the container (default: false, or set via BUILD_ONLY environment variable)" From 4206c733a314c20fdbc318250c0b2327635c7d91 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Sat, 25 Apr 2026 04:50:37 +0800 Subject: [PATCH 129/167] Fix: init missing PlannedStmt fields in orca Those fields are missed by orca which are needed by the pg_stat_statements to identify the query. Without initialization of those fields, pg_stat_statements won't track those queries. --- src/backend/optimizer/plan/orca.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/optimizer/plan/orca.c b/src/backend/optimizer/plan/orca.c index 514385cc2e9..a083fcda6c7 100644 --- a/src/backend/optimizer/plan/orca.c +++ b/src/backend/optimizer/plan/orca.c @@ -405,6 +405,10 @@ optimize_query(Query *parse, int cursorOptions, ParamListInfo boundParams, Optim result->oneoffPlan = glob->oneoffPlan; result->transientPlan = glob->transientPlan; + result->queryId = parse->queryId; + result->stmt_location = parse->stmt_location; + result->stmt_len = parse->stmt_len; + return result; } From 0cda5df4452ba432da9fd5a6ff61b6ac345437f7 Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:55:56 +0300 Subject: [PATCH 130/167] Add Rocky Linux 10 Docker containers (#1676) This is the copy or Rocky9 containers. Changes comparing to Rocky9: - Get rid here of packages not available right now under Rocky10 repositories (for example rocky-release-hpc) - Move to the Java 21, it's default to Rocky Linux 10 Co-authored-by: Leonid Borchuk --- .../docker-cbdb-build-containers.yml | 5 +- .../workflows/docker-cbdb-test-containers.yml | 5 +- devops/deploy/docker/build/rocky10/Dockerfile | 216 +++++++++++++++++ .../build/rocky10/configs/90-cbdb-limits | 32 +++ .../build/rocky10/configs/gpinitsystem.conf | 89 +++++++ .../build/rocky10/configs/init_system.sh | 192 +++++++++++++++ .../build/rocky10/tests/requirements.txt | 3 + .../tests/testinfra/test_cloudberry_db_env.py | 127 ++++++++++ devops/deploy/docker/test/rocky10/Dockerfile | 135 +++++++++++ .../test/rocky10/configs/90-cbdb-limits | 32 +++ .../test/rocky10/configs/gpinitsystem.conf | 87 +++++++ .../test/rocky10/configs/init_system.sh | 221 ++++++++++++++++++ pom.xml | 3 +- 13 files changed, 1144 insertions(+), 3 deletions(-) create mode 100644 devops/deploy/docker/build/rocky10/Dockerfile create mode 100644 devops/deploy/docker/build/rocky10/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf create mode 100755 devops/deploy/docker/build/rocky10/configs/init_system.sh create mode 100644 devops/deploy/docker/build/rocky10/tests/requirements.txt create mode 100644 devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py create mode 100644 devops/deploy/docker/test/rocky10/Dockerfile create mode 100644 devops/deploy/docker/test/rocky10/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf create mode 100755 devops/deploy/docker/test/rocky10/configs/init_system.sh diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index d42139d0fae..538b4e9b179 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -60,6 +60,7 @@ on: paths: - 'devops/deploy/docker/build/rocky8/**' - 'devops/deploy/docker/build/rocky9/**' + - 'devops/deploy/docker/build/rocky10/**' - 'devops/deploy/docker/build/ubuntu22.04/**' - 'devops/deploy/docker/build/ubuntu24.04/**' pull_request: @@ -81,7 +82,7 @@ jobs: # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 strategy: matrix: - platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] + platform: ['rocky8', 'rocky9', 'rocky10', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code with full history @@ -108,6 +109,8 @@ jobs: - 'devops/deploy/docker/build/rocky8/**' rocky9: - 'devops/deploy/docker/build/rocky9/**' + rocky10: + - 'devops/deploy/docker/build/rocky10/**' ubuntu22.04: - 'devops/deploy/docker/build/ubuntu22.04/**' ubuntu24.04: diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 36d320f0737..4d0fb8def33 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -49,6 +49,7 @@ on: paths: - 'devops/deploy/docker/test/rocky8/**' - 'devops/deploy/docker/test/rocky9/**' + - 'devops/deploy/docker/test/rocky10/**' - 'devops/deploy/docker/test/ubuntu22.04/**' - 'devops/deploy/docker/test/ubuntu24.04/**' pull_request: @@ -68,7 +69,7 @@ jobs: strategy: matrix: # Build for Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 - platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] + platform: ['rocky8', 'rocky9', 'rocky10', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code @@ -92,6 +93,8 @@ jobs: - 'devops/deploy/docker/test/rocky8/**' rocky9: - 'devops/deploy/docker/test/rocky9/**' + rocky10: + - 'devops/deploy/docker/test/rocky10/**' ubuntu22.04: - 'devops/deploy/docker/test/ubuntu22.04/**' ubuntu24.04: diff --git a/devops/deploy/docker/build/rocky10/Dockerfile b/devops/deploy/docker/build/rocky10/Dockerfile new file mode 100644 index 00000000000..5cdaaddf9e6 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/Dockerfile @@ -0,0 +1,216 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (Incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Build Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Rocky Linux 10-based container for building +# and developing Apache Cloudberry. It installs necessary system +# utilities, development tools, and configures the environment for SSH +# access and systemd support. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential development tools and libraries installation +# - User configuration for 'gpadmin' with sudo privileges +# +# Usage: +# docker build -t cloudberry-db-env . +# docker run -h cdw -it cloudberry-db-env +# -------------------------------------------------------------------- + +# Base image: Rocky Linux 10 +FROM rockylinux/rockylinux:10 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="America/Los_Angeles" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- +# Install various development tools, system utilities, and libraries +# required for building and running Apache Cloudberry. +# - EPEL repository is enabled for additional packages. +# - Cleanup steps are added to reduce image size after installation. +# -------------------------------------------------------------------- +RUN dnf makecache && \ + dnf install -y \ + epel-release \ + git && \ + dnf makecache && \ + dnf config-manager --disable epel && \ + dnf install -y --enablerepo=epel \ + bat \ + libssh2-devel \ + python3-devel \ + htop && \ + dnf install -y \ + bison \ + cmake3 \ + ed \ + file \ + flex \ + gcc \ + gcc-c++ \ + gdb \ + glibc-langpack-en \ + glibc-locale-source \ + initscripts \ + iproute \ + less \ + lsof \ + m4 \ + net-tools \ + openssh-clients \ + openssh-server \ + perl \ + rpm-build \ + rpmdevtools \ + rsync \ + sudo \ + tar \ + unzip \ + util-linux-ng \ + wget \ + sshpass \ + which && \ + dnf install -y \ + apr-devel \ + bzip2-devel \ + java-21-openjdk \ + java-21-openjdk-devel \ + krb5-devel \ + libcurl-devel \ + libevent-devel \ + libxml2-devel \ + libuuid-devel \ + libzstd-devel \ + lz4 \ + lz4-devel \ + openldap-devel \ + openssl-devel \ + pam-devel \ + perl-ExtUtils-Embed \ + perl-Test-Simple \ + perl-core \ + python3-setuptools \ + readline-devel \ + zlib-devel && \ + dnf install -y --enablerepo=crb \ + liburing-devel \ + libuv-devel \ + libyaml-devel \ + perl-IPC-Run \ + python3-wheel \ + protobuf-devel && \ + dnf clean all && \ + cd && XERCES_LATEST_RELEASE=3.3.0 && \ + wget -nv "https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + echo "$(curl -sL https://archive.apache.org/dist/xerces/c/3/sources/xerces-c-${XERCES_LATEST_RELEASE}.tar.gz.sha256)" | sha256sum -c - && \ + tar xf "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz"; rm "xerces-c-${XERCES_LATEST_RELEASE}.tar.gz" && \ + cd xerces-c-${XERCES_LATEST_RELEASE} && \ + ./configure --prefix=/usr/local/xerces-c && \ + make -j$(nproc) && \ + make install -C ~/xerces-c-${XERCES_LATEST_RELEASE} && \ + rm -rf ~/xerces-c* && \ + cd && GO_VERSION="go1.24.13" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="74d97be1cc3a474129590c67ebf748a96e72d9f3a2b6fef3ed3275de591d49b3"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="1fc94b57134d51669c72173ad5d49fd62afb0f1db9bf3f798fd98ee423f8d730"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Create and configure the 'gpadmin' user with sudo privileges. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- + +# Copy configuration files from their respective locations +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + sed -i.bak -r 's/^(session\s+required\s+pam_limits.so)/#\1/' /etc/pam.d/* && \ + cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime && \ + chmod 777 /tmp/init_system.sh && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ + setcap cap_net_raw+ep /usr/bin/ping && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cbdb/cloudberry-env.sh ]; then\n source /usr/local/cbdb/cloudberry-env.sh\nfi' >> /home/gpadmin/.bashrc && \ + ssh-keygen -A && \ + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" | tee /etc/locale.conf && \ + dnf clean all # Final cleanup to remove unnecessary files + +# Install testinfra via pip +RUN pip3 install pytest-testinfra + +# Copying test files into the container +COPY ./tests /tests + +# -------------------------------------------------------------------- +# Set the Default User and Command +# -------------------------------------------------------------------- +# The default user is set to 'gpadmin', and the container starts by +# running the init_system.sh script. The container also mounts the +# /sys/fs/cgroup volume for systemd compatibility. +# -------------------------------------------------------------------- +USER gpadmin + +VOLUME [ "/sys/fs/cgroup" ] +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits b/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits new file mode 100644 index 00000000000..474957c42f6 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf b/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf new file mode 100644 index 00000000000..d4d312231c5 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/gpinitsystem.conf @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/rocky10/configs/init_system.sh b/devops/deploy/docker/build/rocky10/configs/init_system.sh new file mode 100755 index 00000000000..d8c4a00b035 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/configs/init_system.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/devops/deploy/docker/build/rocky10/tests/requirements.txt b/devops/deploy/docker/build/rocky10/tests/requirements.txt new file mode 100644 index 00000000000..b9711eddac5 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/tests/requirements.txt @@ -0,0 +1,3 @@ +testinfra +pytest-testinfra +paramiko diff --git a/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 00000000000..445318f5335 --- /dev/null +++ b/devops/deploy/docker/build/rocky10/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,127 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "epel-release", + "git", + "bat", + "htop", + "bison", + "cmake", + "gcc", + "gcc-c++", + "glibc-langpack-en", + "glibc-locale-source", + "openssh-clients", + "openssh-server", + "sudo", + "rsync", + "wget", + "openssl-devel", + "python3-devel", + "readline-devel", + "zlib-ng-compat-devel", + "libcurl-devel", + "libevent-devel", + "libxml2-devel", + "libuuid-devel", + "libzstd-devel", + "lz4", + "openldap-devel", + "libuv-devel", + "libyaml-devel" + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "wheel" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_locale_configured(host): + """ + Test if the locale is configured correctly. + """ + locale_conf = host.file("/etc/locale.conf") + assert locale_conf.exists + assert locale_conf.contains("LANG=en_US.UTF-8") + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o777 + + +def test_custom_configuration_files(host): + """ + Test if custom configuration files are correctly copied. + """ + config_file = host.file("/tmp/90-cbdb-limits") + assert config_file.exists + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout diff --git a/devops/deploy/docker/test/rocky10/Dockerfile b/devops/deploy/docker/test/rocky10/Dockerfile new file mode 100644 index 00000000000..ec6b268f708 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/Dockerfile @@ -0,0 +1,135 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (Incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Rocky Linux 10-based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Cloudberry +# Database RPM can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +# Base image: Rocky Linux 10 +FROM rockylinux/rockylinux:10 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="America/Los_Angeles" + +# Environment variables for locale +ENV LANG=en_US.UTF-8 + +# -------------------------------------------------------------------- +# System Update and Installation +# -------------------------------------------------------------------- +# Update the system and install essential system utilities required for +# running and testing Apache Cloudberry. Cleanup the DNF cache afterward +# to reduce the image size. +# -------------------------------------------------------------------- +RUN dnf install -y \ + file \ + gdb \ + glibc-locale-source \ + make \ + openssh \ + openssh-clients \ + openssh-server \ + procps-ng \ + sudo \ + which \ + && \ + dnf clean all # Clean up DNF cache after package installations + +# -------------------------------------------------------------------- +# User Creation and Configuration +# -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd gpadmin -g gpadmin -G wheel && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + echo -e '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + sed -i.bak -r 's/^(session\s+required\s+pam_limits.so)/#\1/' /etc/pam.d/* && \ + cat /usr/share/zoneinfo/${TIMEZONE_VAR} > /etc/localtime && \ + chmod 777 /tmp/init_system.sh && \ + setcap cap_net_raw+ep /usr/bin/ping && \ + ssh-keygen -A && \ + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 && \ + echo "LANG=en_US.UTF-8" | tee /etc/locale.conf + +# -------------------------------------------------------------------- +# Set the Default User and Command +# -------------------------------------------------------------------- +# The default user is set to 'gpadmin', and the container starts by +# running the init_system.sh script. This container serves as a base +# environment, and the Apache Cloudberry RPM can be installed for +# testing and functional verification. +# -------------------------------------------------------------------- +USER gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits b/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits new file mode 100644 index 00000000000..474957c42f6 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf b/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf new file mode 100644 index 00000000000..3dcd5a99365 --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/gpinitsystem.conf @@ -0,0 +1,87 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# ---------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- diff --git a/devops/deploy/docker/test/rocky10/configs/init_system.sh b/devops/deploy/docker/test/rocky10/configs/init_system.sh new file mode 100755 index 00000000000..3ea7e34b0ff --- /dev/null +++ b/devops/deploy/docker/test/rocky10/configs/init_system.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if rpm -q apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/pom.xml b/pom.xml index 1faa566fcec..35450fa0b64 100644 --- a/pom.xml +++ b/pom.xml @@ -1750,7 +1750,8 @@ code or new licensing patterns. devops/deploy/docker/build/rocky8/tests/requirements.txt devops/deploy/docker/build/rocky9/tests/requirements.txt - devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt + devops/deploy/docker/build/rocky10/tests/requirements.txt + devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt devops/deploy/docker/build/ubuntu24.04/tests/requirements.txt + +# gp_url_tools: Cloudberry extension providing functionality for working with URL addresses + +### Features +`gp_url_tools` is an extension for the Cloudberry database that gives implementation +for functions that encode/decode url/uri. + +### Functions +The extension creates the `url_tools_schema` schema and adds four SQL functions: + +- `url_tools_schema.encode_url`/`.encode_uri` + Encodes a text value for use as a URL/URI component by replacing reserved characters with percent-encoded sequences. + +- `url_tools_schema.decode_url`/`.decode_uri` + Decodes percent-encoded sequences in a URL/URI-encoded text value back to their original characters (human-readable). + +### Usage +```sql +CREATE EXTENSION gp_url_tools; +``` +```sql +SELECT url_tools_schema.encode_url('Hello World'); +``` +```bash + encode_url +─────────────── + Hello%20World +(1 row) +``` +```sql +SELECT url_tools_schema.decode_url('Hello%20World'); +``` +```bash + decode_url +───────────── + Hello World +(1 row) +``` +```sql +SELECT url_tools_schema.encode_uri('https://ru.wikipedia.org/wiki/Greenplum_(компания)'); +``` +```bash + encode_uri +──────────────────────────────────────────────────────────────────────────────────────────── + https://ru.wikipedia.org/wiki/Greenplum_(%D0%BA%D0%BE%D0%BC%D0%BF%D0%B0%D0%BD%D0%B8%D1%8F) +``` +```sql +SELECT url_tools_schema.decode_uri('https://ru.wikipedia.org/wiki/Greenplum_(%D0%BA%D0%BE%D0%BC%D0%BF%D0%B0%D0%BD%D0%B8%D1%8F)'); +``` +```bash + decode_uri +──────────────────────────────────────────────────── + https://ru.wikipedia.org/wiki/Greenplum_(компания) +``` + +### Acknowledgments +Thank you very much for the extension for PostgreSQL: https://github.com/okbob/url_encode, its sources were very useful. diff --git a/gpcontrib/gp_url_tools/gp_url_tools.control b/gpcontrib/gp_url_tools/gp_url_tools.control new file mode 100644 index 00000000000..cb16430ad62 --- /dev/null +++ b/gpcontrib/gp_url_tools/gp_url_tools.control @@ -0,0 +1,6 @@ +# gp_url_tools extension +comment = 'Functions for working with URL-s' +default_version = '1.0' +module_pathname = '$libdir/gp_url_tools' +relocatable = true +trusted = true diff --git a/gpcontrib/gp_url_tools/sql/gp_url_tools--1.0.sql b/gpcontrib/gp_url_tools/sql/gp_url_tools--1.0.sql new file mode 100644 index 00000000000..3b2a773719a --- /dev/null +++ b/gpcontrib/gp_url_tools/sql/gp_url_tools--1.0.sql @@ -0,0 +1,27 @@ +/* gp_url_tools--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_url_tools" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS url_tools_schema; +GRANT USAGE ON SCHEMA url_tools_schema TO public; + +CREATE FUNCTION url_tools_schema.encode_url(text) +RETURNS text +AS 'MODULE_PATHNAME', 'encode_url' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION url_tools_schema.decode_url(text) +RETURNS text +AS 'MODULE_PATHNAME', 'decode_url' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION url_tools_schema.encode_uri(text) +RETURNS text +AS 'MODULE_PATHNAME', 'encode_uri' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION url_tools_schema.decode_uri(text) +RETURNS text +AS 'MODULE_PATHNAME', 'decode_uri' +LANGUAGE C IMMUTABLE STRICT; diff --git a/gpcontrib/gp_url_tools/src/gp_url_tools.c b/gpcontrib/gp_url_tools/src/gp_url_tools.c new file mode 100644 index 00000000000..68f8ca0dc05 --- /dev/null +++ b/gpcontrib/gp_url_tools/src/gp_url_tools.c @@ -0,0 +1,261 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gp_url_tools.c + * + * IDENTIFICATION + * gpcontrib/gp_url_tools/src/gp_url_tools.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "mb/pg_wchar.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(encode_url); +PG_FUNCTION_INFO_V1(decode_url); +PG_FUNCTION_INFO_V1(encode_uri); +PG_FUNCTION_INFO_V1(decode_uri); + +static const unsigned int utf16_low[2] = {0xD800, 0xDC00}; +static const unsigned int utf16_high[2] = {0xDBFF, 0xDFFF}; +static const unsigned int utf16_decode = 0x03FF; +static const unsigned int utf16_decode_base = 0x10000; +static const int utf8_with_percent_length = 3; // Example: '%20 +static const int utf16_with_percent_length = 6; // Example: '%u0430' +static const int utf16_surrogate_pair_length = 12; // Example: '%uD800%uDC00' +static const int utf16_second_codepoint_offset = 8; // '%uD800%uDC00' => ('%uD800%u'.lenght == 8) +static const int utf16_past_first_codepoint_offset = 6; // '%uD800%uDC00' => ('%uD800'.lenght == 6) + +static unsigned char hex_char_to_value(char c) { + if ('0' <= c && c <= '9') + return c - '0'; + if ('A' <= c && c <= 'F') + return c - 'A' + 10; + if ('a' <= c && c <= 'f') + return c - 'a' + 10; + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid hexadecimal digit: \"%c\"", c))); +} + +static bool allowed_character(const char c, const char *unreserved_special) { + return ('0' <= c && c <= '9') || ('A' <= c && c <= 'Z') || + ('a' <= c && c <= 'z') || (strchr(unreserved_special, c) != NULL); +} + +static char *write_character(char *output, const char c) { + *output = c; + return ++output; +} + +static void valid_encoding_length(char *current, char *end, int length) { + Assert(current + length <= end); +} + +static text *encode(text *input, const char *unreserved_special) { + int input_length, output_length; + text *output; + char *cinput, *coutput, *current, *cend; + + // Convert input data for processing + cinput = text_to_cstring(input); + input_length = strlen(cinput); + /* + * Worst case: every input byte becomes '%XX' (3 output chars). + * The +1 accounts for the null terminator + */ + output_length = 3 * input_length + 1; + coutput = palloc(sizeof(*coutput) * output_length); + current = coutput; + cend = coutput + output_length; + + for (int i = 0; i < input_length; ++i) { + if (allowed_character(cinput[i], unreserved_special)) { + // Allowed character => copy it into result string + valid_encoding_length(current, cend, 1); + current = write_character(current, cinput[i]); + } else { + // Percent-encode byte as '%XX' + valid_encoding_length(current, cend, 3); + current += sprintf(current, "%%%02X", (unsigned char)cinput[i]); + } + } + // Terminate result string + valid_encoding_length(current, cend, 1); + current = write_character(current, '\0'); + + output = cstring_to_text(coutput); + pfree(coutput); + return output; +} + +static bool valid_utf16(unsigned int byte, int byte_num) { + return utf16_low[byte_num] <= byte && byte <= utf16_high[byte_num]; +} + +static unsigned int decode_utf16_pair(unsigned int bytes[2]) { + Assert(valid_utf16(bytes[0], 0)); + Assert(valid_utf16(bytes[1], 1)); + + return (utf16_decode_base + ((bytes[0] & utf16_decode) << 10) + + (bytes[1] & utf16_decode)); +} + +/* + * Check whether the sequence starts with a percent-encoded UTF-8 byte (%XX). + * + * A UTF-8 percent-encoded byte starts with '%' followed by exactly two hex + * digits (e.g. "%20", "%D0"). This is distinguished from a UTF-16 sequence + * which starts with '%u' or '%U' (e.g. "%uD83D"). + * + * Requires at least 3 characters: '%' + 2 hex digits. + */ +static bool is_utf8(const char *sequence, int length) { + return utf8_with_percent_length <= length && sequence[0] == '%' && + sequence[1] != 'u' && sequence[1] != 'U'; +} + +/* + * Check whether the sequence starts with a legacy percent-encoded UTF-16 unit + * ('%uXXXX' or '%UXXXX'). Requires at least 6 characters: '%u' + 4 hex digits. + */ +static bool is_utf16(const char *sequence, int length) { + return utf16_with_percent_length <= length && sequence[0] == '%' && + (sequence[1] == 'u' || sequence[1] == 'U'); +} + +static void fetch_utf16(unsigned int *byte, const char *input) { + for (int i = 0; i < 4; ++i) + *byte = ((*byte) << 4) | hex_char_to_value(input[i]); +} + +static text *decode(text *input, const char *unreserved_special) { + int input_length; + text *output; + char *cinput, *coutput, *current; + + cinput = text_to_cstring(input); + input_length = strlen(cinput); + coutput = palloc(sizeof(*coutput) * (input_length + 1)); + current = coutput; + + for (int i = 0; i < input_length;) { + if (cinput[i] == '%') { + // Special character => start process '%XX' sequence of chars + if (is_utf16(cinput + i, input_length - i)) { + unsigned int result = 0; + unsigned int bytes[2] = {}; + unsigned char buffer[10] = {}; + + fetch_utf16(bytes, cinput + i + 2); + + if (valid_utf16(bytes[0], 0)) { + if (input_length - i < utf16_surrogate_pair_length) { + ereport( + ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("invalid sequence: not enough characters " + "to decode UTF-16 symbol from %d position", + i))); + } + + fetch_utf16(bytes + 1, + cinput + i + utf16_second_codepoint_offset); + if (!valid_utf16(bytes[1], 1)) { + ereport( + ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("invalid UTF-16 byte: characters from %d " + "position define invalid UTF-16 symbol", + i + utf16_past_first_codepoint_offset))); + } + + result = decode_utf16_pair(bytes); + i += utf16_surrogate_pair_length; + } else { + result = bytes[0]; + i += utf16_with_percent_length; + } + + unicode_to_utf8((pg_wchar)result, buffer); + memcpy(current, buffer, pg_utf_mblen(buffer)); + current += pg_utf_mblen(buffer); + } else if (is_utf8(cinput + i, input_length - i)) { + current = + write_character(current, (hex_char_to_value(cinput[i + 1]) << 4) | + hex_char_to_value(cinput[i + 2])); + i += 3; + } else { + // '%' starts a special sequence, but there are not enough + // characters left to decode it => error 'incorrect sequence of tokens' + ereport(ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("invalid sequence: not enough characters to " + "decode any UTF-typed symbol from %d position", + i))); + } + } else if (allowed_character(cinput[i], unreserved_special)) { + // Copy an unescaped character that is allowed + current = write_character(current, cinput[i]); + i += 1; + } else { + ereport(ERROR, (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("disallowed characters in URL: \"%c\"", + cinput[i]))); + } + } + current = write_character(current, '\0'); + + output = cstring_to_text(coutput); + pfree(coutput); + return output; +} + +static const char *url_unreserved_special = ".-~_"; + +Datum encode_url(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + PG_RETURN_TEXT_P(encode(PG_GETARG_TEXT_PP(0), url_unreserved_special)); +} + +Datum decode_url(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + PG_RETURN_TEXT_P(decode(PG_GETARG_TEXT_PP(0), url_unreserved_special)); +} + +static const char *uri_unreserved_special = "-_.!~*'();/?:@&=+$,#"; + +Datum encode_uri(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + PG_RETURN_TEXT_P(encode(PG_GETARG_TEXT_PP(0), uri_unreserved_special)); +} + +Datum decode_uri(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + PG_RETURN_TEXT_P(decode(PG_GETARG_TEXT_PP(0), uri_unreserved_special)); +} diff --git a/gpcontrib/gp_url_tools/test/expected/gp_url_tools.out b/gpcontrib/gp_url_tools/test/expected/gp_url_tools.out new file mode 100644 index 00000000000..e35fe76e521 --- /dev/null +++ b/gpcontrib/gp_url_tools/test/expected/gp_url_tools.out @@ -0,0 +1,101 @@ +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_url_tools; +-- end_ignore +SET client_encoding TO UTF8; +-- Basic encode/decode with ASCII and %XX escaping. +SELECT url_tools_schema.encode_url('Hello World'); + encode_url +--------------- + Hello%20World +(1 row) + +SELECT url_tools_schema.decode_url('Hello%20World'); + decode_url +------------- + Hello World +(1 row) + +-- encode_url() should escape reserved URL characters like ':'. +SELECT url_tools_schema.encode_url(unnest) from unnest(string_to_array('http://hu.wikipedia.org/wiki/São_Paulo','/')); + encode_url +------------------ + http%3A + + hu.wikipedia.org + wiki + S%C3%A3o_Paulo +(5 rows) + +-- encode_uri() keeps URI delimiters, decode_uri() reverses UTF-8 %XX escaping. +SELECT url_tools_schema.encode_uri('http://hu.wikipedia.org/wiki/São_Paulo'); + encode_uri +--------------------------------------------- + http://hu.wikipedia.org/wiki/S%C3%A3o_Paulo +(1 row) + +SELECT md5(url_tools_schema.decode_uri('http://hu.wikipedia.org/wiki/S%C3%A3o_Paulo')); + md5 +---------------------------------- + 147ded7d471df9cf050bc13242cbf39e +(1 row) + +-- Legacy UTF-16 %uXXXX decoding for BMP characters. +SELECT md5(url_tools_schema.decode_url('%u6D6A%u82B1%u4E00%u6735%u6735%20%u7B2C8%u96C6%20-%20%u89C6%u9891%u5728%u7EBF%u89C2%u770B%20-%20%u6D6A%u82B1%u4E00%u6735%u6735%20-%20%u8292%u679CTV')); + md5 +---------------------------------- + d155b1f894fcd5540ba5881fb71753e1 +(1 row) + +-- Single UTF-16 surrogate pair should decode to one Unicode character. +SELECT url_tools_schema.decode_url('%uD83D%uDE00'); + decode_url +------------ + 😀 +(1 row) + +-- Surrogate pair should also decode correctly in the middle of a string. +SELECT url_tools_schema.decode_url('hello%uD83D%uDE00world'); + decode_url +------------- + hello😀world +(1 row) + +-- Mixed input: ASCII, UTF-8 %XX, UTF-16 BMP, and UTF-16 surrogate pair. +SELECT url_tools_schema.decode_url('A%20%C3%A3%20%u6D6A%20%uD83D%uDE00'); + decode_url +------------ + A ã 浪 😀 +(1 row) + +-- Truncated surrogate pair should raise an error. +SELECT url_tools_schema.decode_url('%uD83D'); +ERROR: invalid sequence: not enough characters to decode UTF-16 symbol from 0 position +-- High surrogate followed by a non-low-surrogate code unit should fail. +SELECT url_tools_schema.decode_url('%uD83D%u0041'); +ERROR: invalid UTF-16 byte: characters from 6 position define invalid UTF-16 symbol +-- NULL input should propagate to NULL for all four SQL-callable functions. +SELECT url_tools_schema.encode_url(NULL) IS NULL; + ?column? +---------- + t +(1 row) + +SELECT url_tools_schema.decode_url(NULL) IS NULL; + ?column? +---------- + t +(1 row) + +SELECT url_tools_schema.encode_uri(NULL) IS NULL; + ?column? +---------- + t +(1 row) + +SELECT url_tools_schema.decode_uri(NULL) IS NULL; + ?column? +---------- + t +(1 row) + +DROP EXTENSION gp_url_tools; diff --git a/gpcontrib/gp_url_tools/test/sql/gp_url_tools.sql b/gpcontrib/gp_url_tools/test/sql/gp_url_tools.sql new file mode 100644 index 00000000000..cd27be212fe --- /dev/null +++ b/gpcontrib/gp_url_tools/test/sql/gp_url_tools.sql @@ -0,0 +1,41 @@ +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_url_tools; +-- end_ignore +SET client_encoding TO UTF8; + +-- Basic encode/decode with ASCII and %XX escaping. +SELECT url_tools_schema.encode_url('Hello World'); +SELECT url_tools_schema.decode_url('Hello%20World'); + +-- encode_url() should escape reserved URL characters like ':'. +SELECT url_tools_schema.encode_url(unnest) from unnest(string_to_array('http://hu.wikipedia.org/wiki/São_Paulo','/')); + +-- encode_uri() keeps URI delimiters, decode_uri() reverses UTF-8 %XX escaping. +SELECT url_tools_schema.encode_uri('http://hu.wikipedia.org/wiki/São_Paulo'); +SELECT md5(url_tools_schema.decode_uri('http://hu.wikipedia.org/wiki/S%C3%A3o_Paulo')); + +-- Legacy UTF-16 %uXXXX decoding for BMP characters. +SELECT md5(url_tools_schema.decode_url('%u6D6A%u82B1%u4E00%u6735%u6735%20%u7B2C8%u96C6%20-%20%u89C6%u9891%u5728%u7EBF%u89C2%u770B%20-%20%u6D6A%u82B1%u4E00%u6735%u6735%20-%20%u8292%u679CTV')); + +-- Single UTF-16 surrogate pair should decode to one Unicode character. +SELECT url_tools_schema.decode_url('%uD83D%uDE00'); + +-- Surrogate pair should also decode correctly in the middle of a string. +SELECT url_tools_schema.decode_url('hello%uD83D%uDE00world'); + +-- Mixed input: ASCII, UTF-8 %XX, UTF-16 BMP, and UTF-16 surrogate pair. +SELECT url_tools_schema.decode_url('A%20%C3%A3%20%u6D6A%20%uD83D%uDE00'); + +-- Truncated surrogate pair should raise an error. +SELECT url_tools_schema.decode_url('%uD83D'); + +-- High surrogate followed by a non-low-surrogate code unit should fail. +SELECT url_tools_schema.decode_url('%uD83D%u0041'); + +-- NULL input should propagate to NULL for all four SQL-callable functions. +SELECT url_tools_schema.encode_url(NULL) IS NULL; +SELECT url_tools_schema.decode_url(NULL) IS NULL; +SELECT url_tools_schema.encode_uri(NULL) IS NULL; +SELECT url_tools_schema.decode_uri(NULL) IS NULL; + +DROP EXTENSION gp_url_tools; diff --git a/pom.xml b/pom.xml index 35450fa0b64..51a7830d5ae 100644 --- a/pom.xml +++ b/pom.xml @@ -155,6 +155,9 @@ code or new licensing patterns. gpcontrib/diskquota/** + gpcontrib/gp_url_tools/Makefile + gpcontrib/gp_url_tools/gp_url_tools.control + getversion .git-blame-ignore-revs .dir-locals.el From 8416ddd5ac73e987cbf25d5891c3b764ae11849e Mon Sep 17 00:00:00 2001 From: Leonid <63977577+leborchuk@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:20:56 +0300 Subject: [PATCH 153/167] Add CONFIGURE_EXTRA_OPTS to use in specific cloud build (#46) Specific option needed to pass parameters to configure script while building on a specific cloud farms --- .../automation/cloudberry/scripts/configure-cloudberry.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index cc9e7376239..89cc9f721f2 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -53,6 +53,7 @@ # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) +# CONFIGURE_EXTRA_OPTS - Args to pass to configure command # ENABLE_DEBUG - Enable debug build options (true/false, defaults to # false) # @@ -179,7 +180,8 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --with-uuid=e2fs \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=${BUILD_DESTINATION}/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib \ + ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 log_section_end "Configure" # Capture version information From 5820a3a0d03d0dea866db3b685a39453f4039576 Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:38:56 +0300 Subject: [PATCH 154/167] Overwrite the Yezzey version with 1.8.10 (#49) --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 2b5dcadd45b..356939e262a 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 2b5dcadd45b4183a4aa5ab976e50c97f0d4c7057 +Subproject commit 356939e262a03ce6f5fe9c076aaf78c52d499bb9 From f770709012a01f5195cfc249448a115d06d93ab6 Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:17:59 +0300 Subject: [PATCH 155/167] MDB admin counterpatch (#54) * Fix: MDB admin counterpatch * Feat: Update comment and refactoring --- src/backend/storage/ipc/signalfuncs.c | 33 +++++++++++++----------- src/test/regress/expected/privileges.out | 10 +++---- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/backend/storage/ipc/signalfuncs.c b/src/backend/storage/ipc/signalfuncs.c index 753b94752d3..7283d65ac6d 100644 --- a/src/backend/storage/ipc/signalfuncs.c +++ b/src/backend/storage/ipc/signalfuncs.c @@ -75,29 +75,32 @@ pg_signal_backend(int pid, int sig, char *msg) local_beentry = pgstat_fetch_stat_local_beentry_by_pid(pid); - /* Only allow superusers to signal superuser-owned backends. */ - if (superuser_arg(proc->roleId) && !superuser()) + /* + * Only allow superusers to signal superuser-owned backends. Any process + * not advertising a role might have the importance of a superuser-owned + * backend, so treat it that way. + * + * The mdb_admin role is also allowed to signal autovacuum workers and MDB + * service backends, even though they are superuser-owned. + */ + if ((!OidIsValid(proc->roleId) || superuser_arg(proc->roleId)) && + !superuser()) { - Oid role; - char * appname; + Oid mdb_admin_role; + char *appname; if (local_beentry == NULL) { return SIGNAL_BACKEND_NOSUPERUSER; } - role = get_role_oid("mdb_admin", true /*if nodoby created mdb_admin role in this database*/); + mdb_admin_role = get_role_oid("mdb_admin", true); appname = local_beentry->backendStatus.st_appname; - // only allow mdb_admin to kill su queries - if (!is_member_of_role(GetUserId(), role)) { - return SIGNAL_BACKEND_NOSUPERUSER; - } - - if (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER) { - // ok - } else if (appname != NULL && strcmp(appname, "MDB") == 0) { - // ok - } else { + if (!(OidIsValid(mdb_admin_role) && + is_member_of_role(GetUserId(), mdb_admin_role) && + (local_beentry->backendStatus.st_backendType == B_AUTOVAC_WORKER || + (appname != NULL && strcmp(appname, "MDB") == 0)))) + { return SIGNAL_BACKEND_NOSUPERUSER; } } diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out index de323f54114..ee9f8fa1530 100644 --- a/src/test/regress/expected/privileges.out +++ b/src/test/regress/expected/privileges.out @@ -2007,13 +2007,9 @@ END$$; ALTER FUNCTION terminate_nothrow OWNER TO pg_signal_backend; SELECT backend_type FROM pg_stat_activity WHERE CASE WHEN COALESCE(usesysid, 10) = 10 THEN terminate_nothrow(pid) END; - backend_type ------------------------------- - autovacuum launcher - dtx recovery process - logical replication launcher - login monitor -(4 rows) + backend_type +-------------- +(0 rows) ROLLBACK; -- test default ACLs From 525412df2ed9e4b281f73b3e1896a08952062d89 Mon Sep 17 00:00:00 2001 From: Smyatkin Maxim Date: Mon, 10 Aug 2026 20:34:10 +0300 Subject: [PATCH 156/167] Support GPDB varlena layout (#47) Support GPDB varlena layout This commit allows cloudbery use GPDB varlena layout, otherwise upgrading from greenplum is blocked. Additionally bump catalog version and write varlena layout metadata (gpdb6 compatible or native Little Endian) in controldata. This allows us to prohibit using wrong binaries with wrong data format, preventing data corruption. For CI: add legacy varlena option to configure and default it to true for our builds --- configure | 37 +++++++++++++++ configure.ac | 16 +++++++ .../scripts/configure-cloudberry.sh | 1 + src/backend/access/transam/xlog.c | 11 +++++ src/bin/pg_controldata/pg_controldata.c | 2 + src/bin/pg_resetwal/pg_resetwal.c | 3 ++ src/bin/pg_upgrade/controldata.c | 31 +++++++++++++ src/bin/pg_upgrade/pg_upgrade.h | 7 +++ src/include/catalog/pg_control.h | 28 ++++++++++- src/include/pg_config.h.in | 3 ++ src/include/postgres.h | 46 +++++++++++++++---- 11 files changed, 176 insertions(+), 9 deletions(-) diff --git a/configure b/configure index 586e4d0cf6a..2eaf177b03a 100755 --- a/configure +++ b/configure @@ -725,6 +725,7 @@ with_zstd with_yezzey PROTOC with_gp_stats_collector +with_varlena_gpdb_layout with_diskquota with_zstd with_libbz2 @@ -949,6 +950,7 @@ with_libbz2 with_zstd with_diskquota with_gp_stats_collector +with_varlena_gpdb_layout with_yezzey with_rt with_libcurl @@ -11455,6 +11457,41 @@ else fi +# +# varlena_gpdb_layout +# + + + +# Check whether --with-varlena_gpdb_layout was given. +if test "${with_varlena_gpdb_layout+set}" = set; then :\ + withval=$with_varlena_gpdb_layout; + case $withval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --with-varlena_gpdb_layout option" "$LINENO" 5 + ;; + esac + +else $as_nop + with_varlena_gpdb_layout=no + +fi + + + + +if test "$with_varlena_gpdb_layout" = yes; then + printf "%s\n" "#define FORCE_BIGENDIAN_VARLENA 1" >>confdefs.h + CFLAGS="$CFLAGS -DFORCE_BIGENDIAN_VARLENA" + CXXFLAGS="$CXXFLAGS -DFORCE_BIGENDIAN_VARLENA" +fi + # # yezzey # diff --git a/configure.ac b/configure.ac index 58fa651ee2e..2b70af608c7 100644 --- a/configure.ac +++ b/configure.ac @@ -1419,6 +1419,22 @@ PGAC_ARG_BOOL(with, gp_stats_collector, yes, [build with gp_stats_collector extension]) AC_SUBST(with_gp_stats_collector) + +# +# varlena_gpdb_layout +# +PGAC_ARG_BOOL(with, varlena_gpdb_layout, no, + [force varlena headers to use legacy gpdb layout]) +AC_SUBST(with_varlena_gpdb_layout) + +if test "$with_varlena_gpdb_layout" = yes; then + AC_DEFINE(FORCE_BIGENDIAN_VARLENA, 1, + [Define to use GPDB6-compatible big-endian/network-order varlena headers.]) + # Keep -D in both flag sets for any TU that does not include pg_config.h. + CFLAGS="$CFLAGS -DFORCE_BIGENDIAN_VARLENA" + CXXFLAGS="$CXXFLAGS -DFORCE_BIGENDIAN_VARLENA" +fi + # # Realtime library # diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 89cc9f721f2..7ec0302821d 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -181,6 +181,7 @@ execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ ${CONFIGURE_MDBLOCALES_OPTS} \ --with-includes=/usr/local/xerces-c/include \ --with-libraries=${BUILD_DESTINATION}/lib \ + --with-varlena_gpdb_layout \ ${CONFIGURE_EXTRA_OPTS:-""} || exit 4 log_section_end "Configure" diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 17ebce7303e..89c2f33dc86 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -4784,6 +4784,8 @@ WriteControlFile(void) ControlFile->float8ByVal = FLOAT8PASSBYVAL; + ControlFile->bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; + /* Contents are protected with a CRC */ INIT_CRC32C(ControlFile->crc); COMP_CRC32C(ControlFile->crc, @@ -5001,6 +5003,15 @@ ReadControlFile(void) errhint("It looks like you need to recompile or initdb."))); #endif + if (ControlFile->bigendian_varlena != BIGENDIAN_VARLENA_LAYOUT) + ereport(FATAL, + (errmsg("database files are incompatible with server"), + errdetail("The database cluster was initialized with %s varlena headers," + " but the server was compiled for %s.", + varlena_order_to_str(ControlFile->bigendian_varlena), + varlena_order_to_str(BIGENDIAN_VARLENA_LAYOUT)), + errhint("It looks like you need to recompile or initdb."))); + wal_segment_size = ControlFile->xlog_seg_size; if (!IsValidWalSegSize(wal_segment_size)) diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index ca5d3de023d..3253f746930 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -336,6 +336,8 @@ main(int argc, char *argv[]) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile->float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + varlena_order_to_str(ControlFile->bigendian_varlena)); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); printf(_("Mock authentication nonce: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 7b660a75e49..7ab0aafa408 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -900,6 +900,7 @@ GuessControlValues(void) ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE; ControlFile.loblksize = LOBLKSIZE; ControlFile.float8ByVal = FLOAT8PASSBYVAL; + ControlFile.bigendian_varlena = BIGENDIAN_VARLENA_LAYOUT; ControlFile.data_checksum_version = PG_DATA_CHECKSUM_VERSION; /* @@ -984,6 +985,8 @@ PrintControlValues(bool guessed) _("64-bit integers")); printf(_("Float8 argument passing: %s\n"), (ControlFile.float8ByVal ? _("by value") : _("by reference"))); + printf(_("Varlena header byte order: %s\n"), + varlena_order_to_str(ControlFile.bigendian_varlena)); printf(_("Data page checksum version: %u\n"), ControlFile.data_checksum_version); printf(_("File encryption method: %s\n"), diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 2b300819678..e51bbb3d3d3 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -66,6 +66,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) bool got_toast = false; bool got_large_object = false; bool got_date_is_int = false; + bool got_varlena_layout = false; bool got_data_checksum_version = false; bool got_cluster_state = false; int got_file_encryption_method = false; @@ -544,6 +545,17 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.date_is_int = strstr(p, "64-bit integers") != NULL; got_date_is_int = true; } + else if ((p = strstr(bufin, "Varlena header byte order:")) != NULL) + { + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + p++; /* remove ':' char */ + cluster->controldata.varlena_bigendian = strstr(p, "network") != NULL; + got_varlena_layout = true; + } else if ((p = strstr(bufin, "checksum")) != NULL) { p = strchr(p, ':'); @@ -645,6 +657,8 @@ get_control_data(ClusterInfo *cluster, bool live_check) !got_index || /* !got_toast || */ (!got_large_object && cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) || + (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) || !got_date_is_int || !got_data_checksum_version || !got_file_encryption_method) { @@ -713,6 +727,10 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) pg_log(PG_REPORT, " large-object chunk size\n"); + if (!got_varlena_layout && + cluster->controldata.ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER) + pg_log(PG_REPORT, " varlena header byte order\n"); + if (!got_date_is_int) pg_log(PG_REPORT, " dates/times are integers?\n"); @@ -769,6 +787,19 @@ check_control_data(ControlData *oldctrl, oldctrl->large_object != newctrl->large_object) pg_fatal("old and new pg_controldata large-object chunk sizes are invalid or do not match\n"); + /* + * On-disk varlena header layout (bigendian_varlena) is recorded in + * pg_control only since VARLENA_LAYOUT_PG_CONTROL_VER. Older clusters + * (GPDB6, pre-feature Cloudberry) don't report it, so only enforce the + * match when the old cluster actually carries the field. A mismatch means + * every variable-length datum would be misread, so this must be fatal. + */ + if (oldctrl->ctrl_ver >= VARLENA_LAYOUT_PG_CONTROL_VER && + oldctrl->varlena_bigendian != newctrl->varlena_bigendian) + pg_fatal("old and new clusters use different on-disk varlena header layouts\n" + "The new cluster must be built to match the old cluster's layout\n" + "(rebuild with or without -DFORCE_BIGENDIAN_VARLENA to match).\n"); + /* * GPDB, since 9.5, pg_upgrade removed the support for 8.3, however, GPDB * still keep it to support upgrading from GPDB 5 diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 2938ea76845..256d07d4db2 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -193,6 +193,12 @@ typedef enum */ #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942 +/* + * varlena header byte order (bigendian_varlena) added to pg_control in + * Cloudberry to guard GPDB6-compatible vs native on-disk varlena layout. + */ +#define VARLENA_LAYOUT_PG_CONTROL_VER 13000701 + /* * change in JSONB format during 9.4 beta */ @@ -323,6 +329,7 @@ typedef struct uint32 large_object; bool date_is_int; bool float8_pass_by_value; + bool varlena_bigendian; uint32 data_checksum_version; int file_encryption_method; } ControlData; diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index c3f87cfd14c..d3252200999 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -27,11 +27,31 @@ * The first four digits is the PostgreSQL version number. The last * four digits indicates the GPDB version. */ -#define PG_CONTROL_VERSION 13000700 +#define PG_CONTROL_VERSION 13000701 /* Nonce key length, see below */ #define MOCK_AUTH_NONCE_LEN 32 +/* + * On-disk varlena header byte order produced by this build, stamped into + * ControlFileData.bigendian_varlena. true == big-endian/network order (real + * big-endian hardware, or -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility; + * see the varlena macros in postgres.h); false == upstream native little-endian. + * Defined here rather than in postgres.h so the frontend tools (pg_resetwal, + * pg_controldata, pg_upgrade) that only include pg_control.h can see it. + */ +#if defined(WORDS_BIGENDIAN) || defined(FORCE_BIGENDIAN_VARLENA) +#define BIGENDIAN_VARLENA_LAYOUT true +#else +#define BIGENDIAN_VARLENA_LAYOUT false +#endif + +static inline const char * +varlena_order_to_str(bool order_is_bigendian) +{ + return order_is_bigendian ? "network-byte-order (GPDB6-compatible)" : "native"; +} + /* * Body of CheckPoint XLOG records. This is declared here because we keep * a copy of the latest one in pg_control for possible disaster recovery. @@ -230,6 +250,12 @@ typedef struct ControlFileData bool float8ByVal; /* float8, int8, etc pass-by-value? */ + /* + * On-disk varlena header byte order (GPDB6-compatible big-endian/network + * order vs upstream native). See BIGENDIAN_VARLENA_LAYOUT above. + */ + bool bigendian_varlena; + /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index b037fc11501..1233958cab4 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -1127,6 +1127,9 @@ # endif #endif +/* Define to use GPDB6-compatible big-endian/network-order varlena headers. */ +#undef FORCE_BIGENDIAN_VARLENA + /* Size of a WAL file block. This need have no particular relation to BLCKSZ. XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O, XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O diff --git a/src/include/postgres.h b/src/include/postgres.h index 90fd1f29d39..92b2e13c07d 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -211,12 +211,15 @@ typedef struct * the specific type and length of the pointer datum. * * NOTE: - * Greenplum differs from PostgreSQL here... In Postgres, it use different - * macros for big-endian and little-endian machines, so the length is contiguous, - * while the 4 byte lengths are stored in native endian format. + * PostgreSQL uses different macros for big-endian and little-endian machines: + * flag bits sit in the physically first byte in both cases, and 4-byte lengths + * are stored in native endian format. * - * Greenplum stored the 4 byte varlena header in network byte order, so it always - * look big-endian in the tuple. + * GPDB 6 (and earlier) stored the 4-byte varlena header in network byte order + * unconditionally (htonl), so it always looked big-endian in the tuple. Stock + * Cloudberry on little-endian follows upstream's native layout. Building with + * -DFORCE_BIGENDIAN_VARLENA restores the GPDB6 encoding for binary pg_upgrade + * compatibility (see below). * */ @@ -229,7 +232,22 @@ typedef struct * checking for IS_1B. */ -#ifdef WORDS_BIGENDIAN +#if defined(FORCE_BIGENDIAN_VARLENA) || defined(WORDS_BIGENDIAN) + +/* + * Big-endian on-disk layout (real big-endian hardware, or little-endian with + * -DFORCE_BIGENDIAN_VARLENA for GPDB6 compatibility). + * + * Flag bits and 1-byte (short) / external headers are byte-level and shared. + * On big-endian hardware the native uint32 already stores the 4-byte length in + * network byte order, so no swap is needed. On little-endian with + * FORCE_BIGENDIAN_VARLENA the native uint32 is not in network order, so the + * 4-byte length word must be swapped with htonl/ntohl. That also makes newly + * written data GPDB6-format, so the entire cluster must be built this way for + * self-consistency. The compressed second word (va_tcinfo) needs no swap: + * GPDB6 stored it natively as well, and its compression-method bits are 0 + * (== PGLZ), which is what Cloudberry expects. + */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x80) == 0x00) @@ -245,17 +263,29 @@ typedef struct (*((uint8 *) (PTR)) != 0) /* VARSIZE_4B() should only be used on known-aligned data */ +#ifdef FORCE_BIGENDIAN_VARLENA +#define VARSIZE_4B(PTR) \ + (ntohl(((varattrib_4b *) (PTR))->va_4byte.va_header) & 0x3FFFFFFF) +#else /* WORDS_BIGENDIAN */ #define VARSIZE_4B(PTR) \ (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) +#endif #define VARSIZE_1B(PTR) \ (((varattrib_1b *) (PTR))->va_header & 0x7F) #define VARTAG_1B_E(PTR) \ (((varattrib_1b_e *) (PTR))->va_tag) +#ifdef FORCE_BIGENDIAN_VARLENA +#define SET_VARSIZE_4B(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl((len) & 0x3FFFFFFF)) +#define SET_VARSIZE_4B_C(PTR,len) \ + (((varattrib_4b *) (PTR))->va_4byte.va_header = htonl(((len) & 0x3FFFFFFF) | 0x40000000)) +#else /* WORDS_BIGENDIAN */ #define SET_VARSIZE_4B(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) #define SET_VARSIZE_4B_C(PTR,len) \ (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) +#endif #define SET_VARSIZE_1B(PTR,len) \ (((varattrib_1b *) (PTR))->va_header = (len) | 0x80) #define SET_VARTAG_1B_E(PTR,tag) \ @@ -263,7 +293,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)(VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) | 0x80) -#else /* !WORDS_BIGENDIAN */ +#else /* !WORDS_BIGENDIAN && !FORCE_BIGENDIAN_VARLENA */ #define VARATT_IS_4B(PTR) \ ((((varattrib_1b *) (PTR))->va_header & 0x01) == 0x00) @@ -297,7 +327,7 @@ typedef struct ((varattrib_1b_e *) (PTR))->va_tag = (tag)) #define VARSIZE_TO_SHORT(PTR) ((char)((VARSIZE(PTR)-VARHDRSZ+VARHDRSZ_SHORT) << 1) | 0x01) -#endif /* WORDS_BIGENDIAN */ +#endif /* WORDS_BIGENDIAN / FORCE_BIGENDIAN_VARLENA */ #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) From 29ae4b5f5d7c60f02a31e9fa4090322db4360bd3 Mon Sep 17 00:00:00 2001 From: Alena Rybakina <58230554+Alena0704@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:34:02 +0800 Subject: [PATCH 157/167] Feature: add TRY_CONVERT extension (#51) TRY_CONVERT(source_value, default_value) casts source_value to the type of default_value and returns default_value whenever the cast fails, which is the behaviour of TRY_CAST in SQL Server. Without it a single malformed value makes the whole query fail, and the plpgsql workarounds that catch the error per row are several times slower. TRY_CONVERT('42'::text, NULL::int2) -- returns 42::int2 TRY_CONVERT('42d'::text, NULL::int2) -- returns NULL::int2 TRY_CONVERT('42d'::text, 1234::int2) -- returns 1234::int2 The conversion to use is resolved the same way the parser resolves an explicit cast in coerce_type(): a pg_cast entry, an I/O conversion or a binary-compatible relabel, followed by the length coercion of the target type. Casts between array types and casts to domain types are not supported and are reported as query errors, the same as a cast that does not exist at all; only a failure caused by the converted data is turned into the default value. The cast itself runs inside a PG_TRY() block, since the datatype input functions of Cloudberry cannot yet report a conversion failure without throwing. The extension is marked trusted, so the owner of a database can install it without being a superuser. See: PR#901 (cherry picked from commit 420a6281ece0703ec3b94c731574204a73669512) Co-authored-by: Vladimir Rachkin --- .github/workflows/build-cloudberry-rocky8.yml | 1 + .github/workflows/build-cloudberry.yml | 1 + .../build-deb-cloudberry-ubuntu24.04.yml | 1 + .github/workflows/build-deb-cloudberry.yml | 1 + GNUmakefile.in | 2 + contrib/Makefile | 1 + contrib/try_convert/.gitignore | 6 + contrib/try_convert/Makefile | 52 ++ contrib/try_convert/README.md | 116 +++ contrib/try_convert/data/corr_date.data | 40 + contrib/try_convert/data/corr_float4.data | 17 + contrib/try_convert/data/corr_float8.data | 17 + contrib/try_convert/data/corr_int2.data | 17 + contrib/try_convert/data/corr_int4.data | 17 + contrib/try_convert/data/corr_int8.data | 17 + contrib/try_convert/data/corr_json.data | 18 + contrib/try_convert/data/corr_numeric.data | 17 + contrib/try_convert/data/corr_time.data | 32 + contrib/try_convert/data/corr_timestamp.data | 32 + .../try_convert/data/corr_timestamptz.data | 32 + contrib/try_convert/data/corr_timetz.data | 32 + contrib/try_convert/data/tt_abstime.data | 12 + contrib/try_convert/data/tt_bit.data | 11 + contrib/try_convert/data/tt_bool.data | 19 + contrib/try_convert/data/tt_bpchar.data | 70 ++ contrib/try_convert/data/tt_char.data | 70 ++ contrib/try_convert/data/tt_cidr.data | 16 + contrib/try_convert/data/tt_citext.data | 70 ++ contrib/try_convert/data/tt_complex.data | 12 + contrib/try_convert/data/tt_date.data | 12 + contrib/try_convert/data/tt_float4.data | 24 + contrib/try_convert/data/tt_float8.data | 30 + contrib/try_convert/data/tt_hstore.data | 12 + contrib/try_convert/data/tt_inet.data | 19 + contrib/try_convert/data/tt_int2.data | 24 + contrib/try_convert/data/tt_int4.data | 27 + contrib/try_convert/data/tt_int8.data | 36 + contrib/try_convert/data/tt_interval.data | 12 + contrib/try_convert/data/tt_json.data | 29 + contrib/try_convert/data/tt_jsonb.data | 29 + contrib/try_convert/data/tt_macaddr.data | 12 + contrib/try_convert/data/tt_money.data | 16 + contrib/try_convert/data/tt_numeric.data | 30 + contrib/try_convert/data/tt_oid.data | 13 + contrib/try_convert/data/tt_point.data | 16 + contrib/try_convert/data/tt_regclass.data | 12 + contrib/try_convert/data/tt_regproc.data | 11 + contrib/try_convert/data/tt_regtype.data | 30 + contrib/try_convert/data/tt_reltime.data | 12 + contrib/try_convert/data/tt_text.data | 70 ++ contrib/try_convert/data/tt_time.data | 12 + contrib/try_convert/data/tt_timestamp.data | 12 + contrib/try_convert/data/tt_timestamptz.data | 12 + contrib/try_convert/data/tt_timetz.data | 12 + contrib/try_convert/data/tt_uuid.data | 12 + contrib/try_convert/data/tt_varbit.data | 11 + contrib/try_convert/data/tt_varchar.data | 70 ++ contrib/try_convert/data/tt_xml.data | 10 + contrib/try_convert/scripts/check_test.py | 87 +++ .../try_convert/scripts/error_safe_check.py | 143 ++++ contrib/try_convert/scripts/find_calls.py | 116 +++ contrib/try_convert/scripts/find_casts.py | 248 +++++++ contrib/try_convert/scripts/find_ereturns.py | 67 ++ contrib/try_convert/scripts/general.py | 136 ++++ contrib/try_convert/scripts/generate_data.py | 119 +++ contrib/try_convert/scripts/generate_test.py | 689 ++++++++++++++++++ contrib/try_convert/scripts/verify.py | 368 ++++++++++ contrib/try_convert/try_convert--1.0.sql | 91 +++ contrib/try_convert/try_convert.c | 564 ++++++++++++++ contrib/try_convert/try_convert.control | 6 + 70 files changed, 4010 insertions(+) create mode 100644 contrib/try_convert/.gitignore create mode 100644 contrib/try_convert/Makefile create mode 100644 contrib/try_convert/README.md create mode 100644 contrib/try_convert/data/corr_date.data create mode 100644 contrib/try_convert/data/corr_float4.data create mode 100644 contrib/try_convert/data/corr_float8.data create mode 100644 contrib/try_convert/data/corr_int2.data create mode 100644 contrib/try_convert/data/corr_int4.data create mode 100644 contrib/try_convert/data/corr_int8.data create mode 100644 contrib/try_convert/data/corr_json.data create mode 100644 contrib/try_convert/data/corr_numeric.data create mode 100644 contrib/try_convert/data/corr_time.data create mode 100644 contrib/try_convert/data/corr_timestamp.data create mode 100644 contrib/try_convert/data/corr_timestamptz.data create mode 100644 contrib/try_convert/data/corr_timetz.data create mode 100644 contrib/try_convert/data/tt_abstime.data create mode 100644 contrib/try_convert/data/tt_bit.data create mode 100644 contrib/try_convert/data/tt_bool.data create mode 100644 contrib/try_convert/data/tt_bpchar.data create mode 100644 contrib/try_convert/data/tt_char.data create mode 100644 contrib/try_convert/data/tt_cidr.data create mode 100644 contrib/try_convert/data/tt_citext.data create mode 100644 contrib/try_convert/data/tt_complex.data create mode 100644 contrib/try_convert/data/tt_date.data create mode 100644 contrib/try_convert/data/tt_float4.data create mode 100644 contrib/try_convert/data/tt_float8.data create mode 100644 contrib/try_convert/data/tt_hstore.data create mode 100644 contrib/try_convert/data/tt_inet.data create mode 100644 contrib/try_convert/data/tt_int2.data create mode 100644 contrib/try_convert/data/tt_int4.data create mode 100644 contrib/try_convert/data/tt_int8.data create mode 100644 contrib/try_convert/data/tt_interval.data create mode 100644 contrib/try_convert/data/tt_json.data create mode 100644 contrib/try_convert/data/tt_jsonb.data create mode 100644 contrib/try_convert/data/tt_macaddr.data create mode 100644 contrib/try_convert/data/tt_money.data create mode 100644 contrib/try_convert/data/tt_numeric.data create mode 100644 contrib/try_convert/data/tt_oid.data create mode 100644 contrib/try_convert/data/tt_point.data create mode 100644 contrib/try_convert/data/tt_regclass.data create mode 100644 contrib/try_convert/data/tt_regproc.data create mode 100644 contrib/try_convert/data/tt_regtype.data create mode 100644 contrib/try_convert/data/tt_reltime.data create mode 100644 contrib/try_convert/data/tt_text.data create mode 100644 contrib/try_convert/data/tt_time.data create mode 100644 contrib/try_convert/data/tt_timestamp.data create mode 100644 contrib/try_convert/data/tt_timestamptz.data create mode 100644 contrib/try_convert/data/tt_timetz.data create mode 100644 contrib/try_convert/data/tt_uuid.data create mode 100644 contrib/try_convert/data/tt_varbit.data create mode 100644 contrib/try_convert/data/tt_varchar.data create mode 100644 contrib/try_convert/data/tt_xml.data create mode 100644 contrib/try_convert/scripts/check_test.py create mode 100644 contrib/try_convert/scripts/error_safe_check.py create mode 100644 contrib/try_convert/scripts/find_calls.py create mode 100644 contrib/try_convert/scripts/find_casts.py create mode 100644 contrib/try_convert/scripts/find_ereturns.py create mode 100644 contrib/try_convert/scripts/general.py create mode 100644 contrib/try_convert/scripts/generate_data.py create mode 100644 contrib/try_convert/scripts/generate_test.py create mode 100644 contrib/try_convert/scripts/verify.py create mode 100644 contrib/try_convert/try_convert--1.0.sql create mode 100644 contrib/try_convert/try_convert.c create mode 100644 contrib/try_convert/try_convert.control diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 9bd5eb7906a..ceb2eb10950 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -310,6 +310,7 @@ jobs: "contrib/pgcrypto:installcheck", "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", + "contrib/try_convert:installcheck", "contrib/passwordcheck:installcheck", "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index baeddaae89f..289592cd405 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -308,6 +308,7 @@ jobs: "contrib/pgcrypto:installcheck", "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", + "contrib/try_convert:installcheck", "contrib/passwordcheck:installcheck", "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml index 041eabc252b..072a0e77258 100644 --- a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml +++ b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml @@ -249,6 +249,7 @@ jobs: "contrib/pgcrypto:installcheck", "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", + "contrib/try_convert:installcheck", "contrib/passwordcheck:installcheck", "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index f865eacba40..3c6b2145719 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -242,6 +242,7 @@ jobs: "contrib/pgcrypto:installcheck", "contrib/pgstattuple:installcheck", "contrib/tablefunc:installcheck", + "contrib/try_convert:installcheck", "contrib/passwordcheck:installcheck", "contrib/pg_aux_catalog:installcheck", "contrib/pg_buffercache:installcheck", diff --git a/GNUmakefile.in b/GNUmakefile.in index 70f635b16e7..50d3be68507 100644 --- a/GNUmakefile.in +++ b/GNUmakefile.in @@ -27,6 +27,7 @@ all: $(MAKE) -C contrib/btree_gin all $(MAKE) -C contrib/pg_trgm all $(MAKE) -C contrib/tablefunc all + $(MAKE) -C contrib/try_convert all $(MAKE) -C contrib/passwordcheck all $(MAKE) -C contrib/pg_buffercache all ifeq ($(with_openssl), yes) @@ -77,6 +78,7 @@ install: $(MAKE) -C contrib/btree_gin $@ $(MAKE) -C contrib/pg_trgm $@ $(MAKE) -C contrib/tablefunc $@ + $(MAKE) -C contrib/try_convert $@ $(MAKE) -C contrib/passwordcheck $@ $(MAKE) -C contrib/pg_buffercache $@ ifeq ($(enable_pax), yes) diff --git a/contrib/Makefile b/contrib/Makefile index 01315b1f6f8..209b47ca213 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -51,6 +51,7 @@ SUBDIRS = \ tablefunc \ tcn \ test_decoding \ + try_convert \ tsm_system_rows \ tsm_system_time \ unaccent \ diff --git a/contrib/try_convert/.gitignore b/contrib/try_convert/.gitignore new file mode 100644 index 00000000000..30d79857f6c --- /dev/null +++ b/contrib/try_convert/.gitignore @@ -0,0 +1,6 @@ +# Generated subdirectories +/results/ +/sql/ +/expected/ +/input/ +/output/ \ No newline at end of file diff --git a/contrib/try_convert/Makefile b/contrib/try_convert/Makefile new file mode 100644 index 00000000000..c30bfca155e --- /dev/null +++ b/contrib/try_convert/Makefile @@ -0,0 +1,52 @@ +# contrib/try_convert/Makefile + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +MODULE_big = try_convert +OBJS = try_convert.o $(WIN32RES) + +EXTENSION = try_convert +DATA = try_convert--1.0.sql +PGFILEDESC = "try_convert - function for type cast" + +# The test cases are derived from the catalog files and from the sample values +# in data/, so they are generated by generate-tests rather than stored in the +# tree, together with the sql/ and expected/ files pg_regress derives from them. +REGRESS = try_convert +REGRESS_PREP = generate-tests +EXTRA_CLEAN = input output sql expected results + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/try_convert +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +.PHONY: generate-tests +generate-tests: + $(MKDIR_P) input output + python3 scripts/generate_test.py + +.PHONY: verify +verify: generate-tests + python3 scripts/verify.py diff --git a/contrib/try_convert/README.md b/contrib/try_convert/README.md new file mode 100644 index 00000000000..e1f20d9bf5f --- /dev/null +++ b/contrib/try_convert/README.md @@ -0,0 +1,116 @@ +# TRY_CONVERT + +TRY_CONVERT is Greenplum/Cloudberry extension, which adds function for error-safe type cast like [TRY_CAST from SQL-Server](https://learn.microsoft.com/ru-ru/sql/t-sql/functions/try-cast-transact-sql?view=sql-server-ver16) + +## Usage + +``` +TRY_CONVERT(SOURCE_VALUE, DEFAULT_VALUE::TARGET_TYPE) + returns (VALUE_IN_TARGET_TYPE or DEFAULT_VALUE) +``` + +``` +TRY_CONVERT('42'::text, NULL::int2) -- returns 42::int2 +TRY_CONVERT('42d'::text, NULL::int2) -- returns NULL::int2 +TRY_CONVERT('42d'::text, 1234::int2) -- returns 1234::int2 +``` + +### Extension's type casts + +Casting from extensions types is able only for extensions: + +- hstore +- citext + +To enable casting from hstore and citext types use, `add_type_for_try_convert(regtype)` function + +## Error handling + +The cast is executed inside a `PG_TRY()` block: when the cast function reports +a failure, the error is discarded and the default value is returned instead. +Query cancellation and assertion failures are never swallowed, they are +re-thrown, the same way plpgsql handles `EXCEPTION WHEN others`. + +The long-term plan is to replace the `PG_TRY()` block by the "soft" error +handling concept introduced in Postgres 17 (https://github.com/postgres/postgres/commit/ccff2d20ed9622815df2a7deffce8a7b14830965), +which lets a datatype input function report a conversion failure without +throwing. That concept was spread on data types in [21be368 +Preview](https://github.com/open-gpdb/gpdb/commit/21be3688729ec4468ffd083da197721860fa2cbd) and [d31f362 +](https://github.com/open-gpdb/gpdb/commit/d31f362250105e456961c2c9249693e42e67eca9) commits. +It requires converting the datatype input functions of Cloudberry first. + +## Why signature is so strange? + +Greenplum/Cloudberry function polymorphism accept to have polymorphic functions only one any type in signature. + +## Supported casts + + ✅ Values Cast + ✅ Types with typemod + ❌ Array-Array Cast + ❌ To Domain type cast + +An unsupported or non-existing cast is a query error, it is not turned into the +default value: only failures caused by the converted data are. + +## Tests + +The regression test is generated out of the catalog files and of the sample +values in `data/`, so it is not stored in the repository. `make installcheck` +generates it into `input/` and `output/` and then runs it: + +``` +make -C contrib/try_convert installcheck +``` + +`make -C contrib/try_convert generate-tests` generates it without running it. + +## Benchmark results by pgbench + + +| | without errors | with errors | +| --- | --- | --- | +| cast | 299.346 | ❌ fails | +| try_convert | 984.280 | 1004.524 | +| sql | 1384.784 | 5787.115 | +| sql execute | 5843.220 | 5898.813 | + + +SQL version: +``` +CREATE OR REPLACE FUNCTION try_convert_into_int(_in text, d int2) RETURNS int2 + LANGUAGE plpgsql AS +$func$ + BEGIN + RETURN CAST(_in AS int2); + EXCEPTION WHEN others THEN + RETURN d; + END +$func$; +``` + + +SQL with execute version: +``` +CREATE OR REPLACE FUNCTION try_convert_by_sql(_in text, INOUT _out ANYELEMENT) + LANGUAGE plpgsql AS +$func$ +BEGIN + EXECUTE format('SELECT %L::%s', $1, pg_typeof(_out)) + INTO _out; +EXCEPTION WHEN others THEN + -- do nothing: _out already carries default +END +$func$; +``` + +Data: +``` +drop table if exists text_ints; create table text_ints (n text); +Insert into text_ints(n) select (random()*1000)::int4::text from generate_series(1,1000000); + +drop table if exists text_error_ints; create table text_error_ints (n text); +Insert into text_error_ints(n) select (random()*1000000)::int8::text from generate_series(1,1000000); +``` + + diff --git a/contrib/try_convert/data/corr_date.data b/contrib/try_convert/data/corr_date.data new file mode 100644 index 00000000000..d11addc0793 --- /dev/null +++ b/contrib/try_convert/data/corr_date.data @@ -0,0 +1,40 @@ +13:35:45 +13:35:90 +13:121:45 +2323:35:45 +13:35:-45 +13:-35:45 +-13:35:45 + +12:52:43 AM +15:52:43 AM +12:152:43 AM +12:52:143 AM +12:52:43 PM +15:52:43 PM +12:152:43 PM +12:52:143 PM + +21:52:07 JST +21:52:07 MSSSSSK + +21:52:07+03:02 +21:52:07+33:02 +21:52:07-33:02 +21:52:07+03:222 + +1982-06-27 18:52:43 +1982-06-32 18:52:43 +1982-32-27 18:52:43 +99999999999999-06-27 18:52:43 + +1982-12-12 01:01:59+09:00 +1982-12-12 01:01:59+29:00 + +1982-06-27 +1982-06-32 +1982-32-27 +999999999999999-06-27 + +11-30-0002 BC +11-30-200000000 BC \ No newline at end of file diff --git a/contrib/try_convert/data/corr_float4.data b/contrib/try_convert/data/corr_float4.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_float4.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_float8.data b/contrib/try_convert/data/corr_float8.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_float8.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_int2.data b/contrib/try_convert/data/corr_int2.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_int2.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_int4.data b/contrib/try_convert/data/corr_int4.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_int4.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_int8.data b/contrib/try_convert/data/corr_int8.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_int8.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_json.data b/contrib/try_convert/data/corr_json.data new file mode 100644 index 00000000000..e4e284fe311 --- /dev/null +++ b/contrib/try_convert/data/corr_json.data @@ -0,0 +1,18 @@ +{ + +[ +{"world":"CC", "query":"AA", "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} +{"world":"CCquery":"AA"} +{,,,,,,} +{"world":"CC",,,,, "query":"AA", "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} +{"world":"CC", "query"::"AA", "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} + +{"world"} +{"world":"CC", "query":["AA":"BB", "sds"], "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} +}}}}}}}} +[[[[[[]]]]]] +{[]} +[{}] +][][][][][][] +{}}{}{}}}{{{}{}{}{}}}} +kfgbidsfgadgbdfiugbhdiug \ No newline at end of file diff --git a/contrib/try_convert/data/corr_numeric.data b/contrib/try_convert/data/corr_numeric.data new file mode 100644 index 00000000000..3d3ada99f23 --- /dev/null +++ b/contrib/try_convert/data/corr_numeric.data @@ -0,0 +1,17 @@ +1 +2 +333333333 +333333333333333333333 +5555555555555555555555555555555555555555555 +dfgdg435fw2342rf445 +6.6332342343243242342342342342343243244324 +354345345345435345345345345.23467567867325345345 +$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ +7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7.7 +Twenty-two +12234441231.13123123 +485728435843759345135.4324234234 +4823849249230.f +19203234723472-e102 +0000000000000.00000000000000000001 +234324+e4443 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_time.data b/contrib/try_convert/data/corr_time.data new file mode 100644 index 00000000000..179fdf1c110 --- /dev/null +++ b/contrib/try_convert/data/corr_time.data @@ -0,0 +1,32 @@ +13:35:45 +13:35:90 +13:121:45 +2323:35:45 +13:35:-45 +13:-35:45 +-13:35:45 + +12:52:43 AM +15:52:43 AM +12:152:43 AM +12:52:143 AM +12:52:43 PM +15:52:43 PM +12:152:43 PM +12:52:143 PM + +21:52:07 JST +21:52:07 MSSSSSK + +21:52:07+03:02 +21:52:07+33:02 +21:52:07-33:02 +21:52:07+03:222 + +1982-06-27 18:52:43 +1982-06-32 18:52:43 +1982-32-27 18:52:43 +99999999999999-06-27 18:52:43 + +1982-12-12 01:01:59+09:00 +1982-12-12 01:01:59+19:00 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_timestamp.data b/contrib/try_convert/data/corr_timestamp.data new file mode 100644 index 00000000000..179fdf1c110 --- /dev/null +++ b/contrib/try_convert/data/corr_timestamp.data @@ -0,0 +1,32 @@ +13:35:45 +13:35:90 +13:121:45 +2323:35:45 +13:35:-45 +13:-35:45 +-13:35:45 + +12:52:43 AM +15:52:43 AM +12:152:43 AM +12:52:143 AM +12:52:43 PM +15:52:43 PM +12:152:43 PM +12:52:143 PM + +21:52:07 JST +21:52:07 MSSSSSK + +21:52:07+03:02 +21:52:07+33:02 +21:52:07-33:02 +21:52:07+03:222 + +1982-06-27 18:52:43 +1982-06-32 18:52:43 +1982-32-27 18:52:43 +99999999999999-06-27 18:52:43 + +1982-12-12 01:01:59+09:00 +1982-12-12 01:01:59+19:00 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_timestamptz.data b/contrib/try_convert/data/corr_timestamptz.data new file mode 100644 index 00000000000..179fdf1c110 --- /dev/null +++ b/contrib/try_convert/data/corr_timestamptz.data @@ -0,0 +1,32 @@ +13:35:45 +13:35:90 +13:121:45 +2323:35:45 +13:35:-45 +13:-35:45 +-13:35:45 + +12:52:43 AM +15:52:43 AM +12:152:43 AM +12:52:143 AM +12:52:43 PM +15:52:43 PM +12:152:43 PM +12:52:143 PM + +21:52:07 JST +21:52:07 MSSSSSK + +21:52:07+03:02 +21:52:07+33:02 +21:52:07-33:02 +21:52:07+03:222 + +1982-06-27 18:52:43 +1982-06-32 18:52:43 +1982-32-27 18:52:43 +99999999999999-06-27 18:52:43 + +1982-12-12 01:01:59+09:00 +1982-12-12 01:01:59+19:00 \ No newline at end of file diff --git a/contrib/try_convert/data/corr_timetz.data b/contrib/try_convert/data/corr_timetz.data new file mode 100644 index 00000000000..179fdf1c110 --- /dev/null +++ b/contrib/try_convert/data/corr_timetz.data @@ -0,0 +1,32 @@ +13:35:45 +13:35:90 +13:121:45 +2323:35:45 +13:35:-45 +13:-35:45 +-13:35:45 + +12:52:43 AM +15:52:43 AM +12:152:43 AM +12:52:143 AM +12:52:43 PM +15:52:43 PM +12:152:43 PM +12:52:143 PM + +21:52:07 JST +21:52:07 MSSSSSK + +21:52:07+03:02 +21:52:07+33:02 +21:52:07-33:02 +21:52:07+03:222 + +1982-06-27 18:52:43 +1982-06-32 18:52:43 +1982-32-27 18:52:43 +99999999999999-06-27 18:52:43 + +1982-12-12 01:01:59+09:00 +1982-12-12 01:01:59+19:00 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_abstime.data b/contrib/try_convert/data/tt_abstime.data new file mode 100644 index 00000000000..5cf22efbee9 --- /dev/null +++ b/contrib/try_convert/data/tt_abstime.data @@ -0,0 +1,12 @@ +2015-10-28 16:35:45 +1974-02-15 15:52:07 +2019-07-22 23:19:33 +2024-06-24 23:02:00 +1986-12-03 12:16:50 +1982-12-11 19:01:59 +1979-07-17 08:59:54 +1982-06-27 18:52:43 +2001-06-25 20:23:38 +1975-02-17 19:50:17 +2007-06-14 13:50:09 +1973-06-11 12:06:52 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_bit.data b/contrib/try_convert/data/tt_bit.data new file mode 100644 index 00000000000..f6724da39ab --- /dev/null +++ b/contrib/try_convert/data/tt_bit.data @@ -0,0 +1,11 @@ +1 +0 +1010101 +11001010 +1011011101111 +01001 +1111 +00000000000000000000000 +1111111111111111111 +0101001010101 +11101001000100 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_bool.data b/contrib/try_convert/data/tt_bool.data new file mode 100644 index 00000000000..78542aab4c2 --- /dev/null +++ b/contrib/try_convert/data/tt_bool.data @@ -0,0 +1,19 @@ +f +true +t +false +t +t +f +t +False +t +f +True +TRUE +f +f +1 +0 +f +FALSE \ No newline at end of file diff --git a/contrib/try_convert/data/tt_bpchar.data b/contrib/try_convert/data/tt_bpchar.data new file mode 100644 index 00000000000..21c17124dfc --- /dev/null +++ b/contrib/try_convert/data/tt_bpchar.data @@ -0,0 +1,70 @@ +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. + +std::to_string + C++ Strings library std::basic_string +Defined in header +std::string to_string( int value ); +std::string to_string( long value ); +std::string to_string( long long value ); +std::string to_string( unsigned value ); +std::string to_string( unsigned long value ); +std::string to_string( unsigned long long value ); +std::string to_string( float value ); +std::string to_string( double value ); +std::string to_string( long double value ); +Converts a numeric value to std::string. + +Let buf be an internal to the conversion functions buffer, sufficiently large to contain the result of conversion. + +1) Converts a signed integer to a string as if by std::sprintf(buf, "%d", value). +2) Converts a signed integer to a string as if by std::sprintf(buf, "%ld", value). +3) Converts a signed integer to a string as if by std::sprintf(buf, "%lld", value). +4) Converts an unsigned integer to a string as if by std::sprintf(buf, "%u", value). +5) Converts an unsigned integer to a string as if by std::sprintf(buf, "%lu", value). +6) Converts an unsigned integer to a string as if by std::sprintf(buf, "%llu", value). +7,8) Converts a floating point value to a string as if by std::sprintf(buf, "%f", value). +9) Converts a floating point value to a string as if by std::sprintf(buf, "%Lf", value). +(until C++26) +1-9) Converts a numeric value to a string as if by std::format("{}", value). +(since C++26) +Parameters +Return value +A string holding the converted value. + +Exceptions +May throw std::bad_alloc from the std::string constructor. + +Notes +With floating point types std::to_string may yield unexpected results as the number of significant digits in the returned string can be zero, see the example. +The return value may differ significantly from what std::cout prints by default, see the example. +std::to_string relies on the current C locale for formatting purposes, and therefore concurrent calls to std::to_string from multiple threads may result in partial serialization of calls. +The results of overloads for integer types do not rely on the current C locale, and thus implementations generally avoid access to the current C locale in these overloads for both correctness and performance. However, such avoidance is not guaranteed by the standard. +(until C++26) +C++17 provides std::to_chars as a higher-performance locale-independent alternative. \ No newline at end of file diff --git a/contrib/try_convert/data/tt_char.data b/contrib/try_convert/data/tt_char.data new file mode 100644 index 00000000000..21c17124dfc --- /dev/null +++ b/contrib/try_convert/data/tt_char.data @@ -0,0 +1,70 @@ +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. + +std::to_string + C++ Strings library std::basic_string +Defined in header +std::string to_string( int value ); +std::string to_string( long value ); +std::string to_string( long long value ); +std::string to_string( unsigned value ); +std::string to_string( unsigned long value ); +std::string to_string( unsigned long long value ); +std::string to_string( float value ); +std::string to_string( double value ); +std::string to_string( long double value ); +Converts a numeric value to std::string. + +Let buf be an internal to the conversion functions buffer, sufficiently large to contain the result of conversion. + +1) Converts a signed integer to a string as if by std::sprintf(buf, "%d", value). +2) Converts a signed integer to a string as if by std::sprintf(buf, "%ld", value). +3) Converts a signed integer to a string as if by std::sprintf(buf, "%lld", value). +4) Converts an unsigned integer to a string as if by std::sprintf(buf, "%u", value). +5) Converts an unsigned integer to a string as if by std::sprintf(buf, "%lu", value). +6) Converts an unsigned integer to a string as if by std::sprintf(buf, "%llu", value). +7,8) Converts a floating point value to a string as if by std::sprintf(buf, "%f", value). +9) Converts a floating point value to a string as if by std::sprintf(buf, "%Lf", value). +(until C++26) +1-9) Converts a numeric value to a string as if by std::format("{}", value). +(since C++26) +Parameters +Return value +A string holding the converted value. + +Exceptions +May throw std::bad_alloc from the std::string constructor. + +Notes +With floating point types std::to_string may yield unexpected results as the number of significant digits in the returned string can be zero, see the example. +The return value may differ significantly from what std::cout prints by default, see the example. +std::to_string relies on the current C locale for formatting purposes, and therefore concurrent calls to std::to_string from multiple threads may result in partial serialization of calls. +The results of overloads for integer types do not rely on the current C locale, and thus implementations generally avoid access to the current C locale in these overloads for both correctness and performance. However, such avoidance is not guaranteed by the standard. +(until C++26) +C++17 provides std::to_chars as a higher-performance locale-independent alternative. \ No newline at end of file diff --git a/contrib/try_convert/data/tt_cidr.data b/contrib/try_convert/data/tt_cidr.data new file mode 100644 index 00000000000..73d05ee9bdc --- /dev/null +++ b/contrib/try_convert/data/tt_cidr.data @@ -0,0 +1,16 @@ +192.168.100.128/25 +192.168/24 +192.168/25 +192.168.1 +192.168 +128.1 +128 +128.1.2 +10.1.2 +10.1 +10 +10.1.2.3/32 +2001:4f8:3:ba::/64 +2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128 +::ffff:1.2.3.0/120 +::ffff:1.2.3.0/128 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_citext.data b/contrib/try_convert/data/tt_citext.data new file mode 100644 index 00000000000..21c17124dfc --- /dev/null +++ b/contrib/try_convert/data/tt_citext.data @@ -0,0 +1,70 @@ +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. + +std::to_string + C++ Strings library std::basic_string +Defined in header +std::string to_string( int value ); +std::string to_string( long value ); +std::string to_string( long long value ); +std::string to_string( unsigned value ); +std::string to_string( unsigned long value ); +std::string to_string( unsigned long long value ); +std::string to_string( float value ); +std::string to_string( double value ); +std::string to_string( long double value ); +Converts a numeric value to std::string. + +Let buf be an internal to the conversion functions buffer, sufficiently large to contain the result of conversion. + +1) Converts a signed integer to a string as if by std::sprintf(buf, "%d", value). +2) Converts a signed integer to a string as if by std::sprintf(buf, "%ld", value). +3) Converts a signed integer to a string as if by std::sprintf(buf, "%lld", value). +4) Converts an unsigned integer to a string as if by std::sprintf(buf, "%u", value). +5) Converts an unsigned integer to a string as if by std::sprintf(buf, "%lu", value). +6) Converts an unsigned integer to a string as if by std::sprintf(buf, "%llu", value). +7,8) Converts a floating point value to a string as if by std::sprintf(buf, "%f", value). +9) Converts a floating point value to a string as if by std::sprintf(buf, "%Lf", value). +(until C++26) +1-9) Converts a numeric value to a string as if by std::format("{}", value). +(since C++26) +Parameters +Return value +A string holding the converted value. + +Exceptions +May throw std::bad_alloc from the std::string constructor. + +Notes +With floating point types std::to_string may yield unexpected results as the number of significant digits in the returned string can be zero, see the example. +The return value may differ significantly from what std::cout prints by default, see the example. +std::to_string relies on the current C locale for formatting purposes, and therefore concurrent calls to std::to_string from multiple threads may result in partial serialization of calls. +The results of overloads for integer types do not rely on the current C locale, and thus implementations generally avoid access to the current C locale in these overloads for both correctness and performance. However, such avoidance is not guaranteed by the standard. +(until C++26) +C++17 provides std::to_chars as a higher-performance locale-independent alternative. \ No newline at end of file diff --git a/contrib/try_convert/data/tt_complex.data b/contrib/try_convert/data/tt_complex.data new file mode 100644 index 00000000000..d7ebf7eb244 --- /dev/null +++ b/contrib/try_convert/data/tt_complex.data @@ -0,0 +1,12 @@ +1.1 + 1.0i +10.0 +-1213 - 13.342i +1546i +0.00004 + 9i +111111 - 0.00i +0i +0 +-111.234 +567.1 +123331 +-2 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_date.data b/contrib/try_convert/data/tt_date.data new file mode 100644 index 00000000000..a933b76310e --- /dev/null +++ b/contrib/try_convert/data/tt_date.data @@ -0,0 +1,12 @@ +2015-10-28 +1974-02-15 +2019-07-22 +2024-06-24 +1986-12-03 +1982-12-12 +1979-07-17 +1982-06-27 +2001-06-25 +1975-02-18 +2007-06-14 +1973-06-11 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_float4.data b/contrib/try_convert/data/tt_float4.data new file mode 100644 index 00000000000..d634a540948 --- /dev/null +++ b/contrib/try_convert/data/tt_float4.data @@ -0,0 +1,24 @@ +5.09526 +0.90909 +0.47116 +1.09649 +62.7446 +79.2079 +42.2159 +6.35277 +381.619 +996.121 +529.114 +971.078 +8607.79 +114.810 +7207.21 +6817.10 +53697.0 +26682.5 +64096.1 +11155.2 +434765 +453723 +953815 +875852 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_float8.data b/contrib/try_convert/data/tt_float8.data new file mode 100644 index 00000000000..441d46d8a57 --- /dev/null +++ b/contrib/try_convert/data/tt_float8.data @@ -0,0 +1,30 @@ +2.6338905075109076 +5.005861130502983 +17.865188053013135 +91.26278393448204 +870.5185698367669 +298.4447914486329 +6389.494948660052 +6089.702114381723 +15283.926854963482 +76251.08000751512 +539379.0301196257 +778626.4786305582 +5303536.721951775 +5718.961279435053 +32415605.70046731 +1947674.2385832302 +929098616.2646171 +878721877.8231843 +8316655293.611794 +3075141254.026614 +5792516649.418756 +87800959920.40405 +946949445297.994 +85653452067.87878 +4859904633166.138 +692125184683.836 +76060216525723.16 +76583442930698.78 +128391464499762.8 +475282378098731.3 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_hstore.data b/contrib/try_convert/data/tt_hstore.data new file mode 100644 index 00000000000..6d21a65218e --- /dev/null +++ b/contrib/try_convert/data/tt_hstore.data @@ -0,0 +1,12 @@ +k => v +foo => bar, baz => whatever +"1-a" => "anything at all" +k => v +foo => bar, baz => whatever +"1-a" => "anything at all" +k => v +foo => bar, baz => whatever +"1-a" => "anything at all" +k => v +foo => bar, baz => whatever +"1-a" => "anything at all" \ No newline at end of file diff --git a/contrib/try_convert/data/tt_inet.data b/contrib/try_convert/data/tt_inet.data new file mode 100644 index 00000000000..e5231166925 --- /dev/null +++ b/contrib/try_convert/data/tt_inet.data @@ -0,0 +1,19 @@ +192.168.100.128/25 +192.168.0.0/24 +192.168.0.0/25 +192.168.100.128 +192.168.0.1/24 +192.168.0.1/25 +192.168.1.0 +192.168.0.0 +128.1.0.0 +128.0.0.0 +128.1.2.0 +10.1.2.0 +10.1.0.0 +10.0.0.0 +10.1.2.3/32 +2001:4f8:3:ba::/64 +2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128 +::ffff:1.2.3.0/120 +::ffff:1.2.3.0/128 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_int2.data b/contrib/try_convert/data/tt_int2.data new file mode 100644 index 00000000000..d6021f2c5e3 --- /dev/null +++ b/contrib/try_convert/data/tt_int2.data @@ -0,0 +1,24 @@ +6 +0 +2 +2 +7 +6 +89 +8 +42 +2 +21 +50 +26 +198 +649 +544 +220 +589 +8094 +64 +8058 +6981 +3402 +1554 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_int4.data b/contrib/try_convert/data/tt_int4.data new file mode 100644 index 00000000000..1508f6a912a --- /dev/null +++ b/contrib/try_convert/data/tt_int4.data @@ -0,0 +1,27 @@ +9 +3 +0 +9 +84 +60 +807 +729 +536 +9731 +3785 +5520 +82940 +61851 +86170 +577352 +704571 +45824 +2278982 +2893879 +797919 +23279088 +10100142 +27797360 +635684444 +364832178 +370180967 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_int8.data b/contrib/try_convert/data/tt_int8.data new file mode 100644 index 00000000000..cb2afdd7cb2 --- /dev/null +++ b/contrib/try_convert/data/tt_int8.data @@ -0,0 +1,36 @@ +2 +2 +93 +64 +609 +171 +7291 +1634 +37945 +98952 +639999 +556949 +6846142 +8428519 +77599991 +22904807 +32100243 +315453048 +2677408759 +2109828435 +94290971433 +87636762647 +314677880798 +655438665294 +3956319010606 +9145475897405 +45885185258739 +26488016649805 +246627507693983 +561368134163150 +2627416085229352 +5845859902235405 +89782288360247696 +39940050514039728 +219320759157283328 +997537606495110272 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_interval.data b/contrib/try_convert/data/tt_interval.data new file mode 100644 index 00000000000..3f44dbbc925 --- /dev/null +++ b/contrib/try_convert/data/tt_interval.data @@ -0,0 +1,12 @@ +16736 days, 13:35:45 +1506 days, 12:52:07 +18099 days, 20:19:33 +19898 days, 20:02:00 +6180 days, 9:16:50 +4727 days, 16:01:59 +3484 days, 5:59:54 +4560 days, 15:52:43 +11498 days, 17:23:38 +1873 days, 16:50:17 +13678 days, 10:50:09 +1257 days, 9:06:52 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_json.data b/contrib/try_convert/data/tt_json.data new file mode 100644 index 00000000000..947eac50831 --- /dev/null +++ b/contrib/try_convert/data/tt_json.data @@ -0,0 +1,29 @@ +{"glossary": {"title": "example glossary","GlossDiv": {"title": "S","GlossList": {"GlossEntry": {"ID": "SGML","SortAs": "SGML","GlossTerm": "Standard Generalized Markup Language","Acronym": "SGML","Abbrev": "ISO 8879:1986","GlossDef": {"para": "A meta-markup language, used to create markup languages such as DocBook.","GlossSeeAlso": ["GML", "XML"]},"GlossSee": "markup"}}}}} +{"menu": {"id": "file","value": "File","popup": {"menuitem": [{"value": "New", "onclick": "CreateNewDoc()"},{"value": "Open", "onclick": "OpenDoc()"},{"value": "Close", "onclick": "CloseDoc()"}]}}} +{"widget": {"debug": "on","window": {"title": "Sample Konfabulator Widget","name": "main_window","width": 500,"height": 500},"image": { "src": "Images/Sun.png","name": "sun1","hOffset": 250,"vOffset": 250,"alignment": "center"},"text": {"data": "Click Here","size": 36,"style": "bold","name": "text1","hOffset": 250,"vOffset": 100,"alignment": "center","onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"}}} +{"menu": {"header": "SVG Viewer","items": [{"id": "Open"},{"id": "OpenNew", "label": "Open New"},null,{"id": "ZoomIn", "label": "Zoom In"},{"id": "ZoomOut", "label": "Zoom Out"},{"id": "OriginalView", "label": "Original View"},null,{"id": "Quality"},{"id": "Pause"},{"id": "Mute"},null,{"id": "Find", "label": "Find..."},{"id": "FindAgain", "label": "Find Again"},{"id": "Copy"},{"id": "CopyAgain", "label": "Copy Again"},{"id": "CopySVG", "label": "Copy SVG"},{"id": "ViewSVG", "label": "View SVG"},{"id": "ViewSource", "label": "View Source"},{"id": "SaveAs", "label": "Save As"},null,{"id": "Help"},{"id": "About", "label": "About Adobe CVG Viewer..."}]}} +{"line":1, "date":"CB", "node":"AA"} +{"cleaned":false, "status":59, "line":2, "disabled":false, "node":"CBB"} +{"indexed":true, "status":35, "line":3, "disabled":false, "wait":"CAA", "subtitle":"BA", "user":"CCA"} +{"line":4, "disabled":true, "space":"BB"} +{"cleaned":false, "line":5, "wait":"BB", "query":"CAC", "coauthors":"ACA", "node":"CBA"} +{"world":"CB", "query":"CBC", "indexed":false, "line":6, "pos":92, "date":"AAB", "space":"CB", "coauthors":"ACA", "node":"CBC"} +{"state":98, "org":43, "line":7, "pos":97} +{"auth":"BB", "title":"CAC", "query":"BA", "status":94, "line":8, "coauthors":"BBB"} +{"auth":"BAC", "title":"CAA", "wait":"CA", "bad":true, "query":"AA", "indexed":true, "line":9, "pos":56} +{"title":"AAC", "bad":true, "user":"AAB", "query":"AC", "line":10, "node":"AB"} +{"world":"CAC", "user":"AB", "query":"ACA", "indexed":true, "line":11, "space":"CB"} +{"line":12, "pos":72, "abstract":"BBA", "space":"AAC"} +{} +{"world":"CC", "query":"AA", "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} +{"org":68, "title":"BBB", "query":"BAC", "line":15, "public":false} +{"org":73, "user":"AA", "indexed":true, "line":16, "date":"CCC", "public":true, "coauthors":"AB"} +{"indexed":false, "line":17} +{"state":23, "auth":"BCC", "org":38, "status":28, "line":18, "disabled":false, "abstract":"CB"} +{"state":99, "auth":"CA", "indexed":true, "line":19, "date":"BA"} +{"wait":"CBA", "user":"BBA", "indexed":true, "line":20, "disabled":false, "abstract":"BA", "date":"ABA"} +{"org":10, "query":"AC", "indexed":false, "line":21, "disabled":true, "abstract":"CA", "pos":44} +{"state":65, "title":"AC", "user":"AAC", "cleaned":true, "status":93, "line":22, "abstract":"ABC", "node":"CCC"} +{"subtitle":"AC", "user":"CCC", "line":23} +{"state":67, "world":"ACB", "bad":true, "user":"CB", "line":24, "disabled":true} +{} \ No newline at end of file diff --git a/contrib/try_convert/data/tt_jsonb.data b/contrib/try_convert/data/tt_jsonb.data new file mode 100644 index 00000000000..947eac50831 --- /dev/null +++ b/contrib/try_convert/data/tt_jsonb.data @@ -0,0 +1,29 @@ +{"glossary": {"title": "example glossary","GlossDiv": {"title": "S","GlossList": {"GlossEntry": {"ID": "SGML","SortAs": "SGML","GlossTerm": "Standard Generalized Markup Language","Acronym": "SGML","Abbrev": "ISO 8879:1986","GlossDef": {"para": "A meta-markup language, used to create markup languages such as DocBook.","GlossSeeAlso": ["GML", "XML"]},"GlossSee": "markup"}}}}} +{"menu": {"id": "file","value": "File","popup": {"menuitem": [{"value": "New", "onclick": "CreateNewDoc()"},{"value": "Open", "onclick": "OpenDoc()"},{"value": "Close", "onclick": "CloseDoc()"}]}}} +{"widget": {"debug": "on","window": {"title": "Sample Konfabulator Widget","name": "main_window","width": 500,"height": 500},"image": { "src": "Images/Sun.png","name": "sun1","hOffset": 250,"vOffset": 250,"alignment": "center"},"text": {"data": "Click Here","size": 36,"style": "bold","name": "text1","hOffset": 250,"vOffset": 100,"alignment": "center","onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"}}} +{"menu": {"header": "SVG Viewer","items": [{"id": "Open"},{"id": "OpenNew", "label": "Open New"},null,{"id": "ZoomIn", "label": "Zoom In"},{"id": "ZoomOut", "label": "Zoom Out"},{"id": "OriginalView", "label": "Original View"},null,{"id": "Quality"},{"id": "Pause"},{"id": "Mute"},null,{"id": "Find", "label": "Find..."},{"id": "FindAgain", "label": "Find Again"},{"id": "Copy"},{"id": "CopyAgain", "label": "Copy Again"},{"id": "CopySVG", "label": "Copy SVG"},{"id": "ViewSVG", "label": "View SVG"},{"id": "ViewSource", "label": "View Source"},{"id": "SaveAs", "label": "Save As"},null,{"id": "Help"},{"id": "About", "label": "About Adobe CVG Viewer..."}]}} +{"line":1, "date":"CB", "node":"AA"} +{"cleaned":false, "status":59, "line":2, "disabled":false, "node":"CBB"} +{"indexed":true, "status":35, "line":3, "disabled":false, "wait":"CAA", "subtitle":"BA", "user":"CCA"} +{"line":4, "disabled":true, "space":"BB"} +{"cleaned":false, "line":5, "wait":"BB", "query":"CAC", "coauthors":"ACA", "node":"CBA"} +{"world":"CB", "query":"CBC", "indexed":false, "line":6, "pos":92, "date":"AAB", "space":"CB", "coauthors":"ACA", "node":"CBC"} +{"state":98, "org":43, "line":7, "pos":97} +{"auth":"BB", "title":"CAC", "query":"BA", "status":94, "line":8, "coauthors":"BBB"} +{"auth":"BAC", "title":"CAA", "wait":"CA", "bad":true, "query":"AA", "indexed":true, "line":9, "pos":56} +{"title":"AAC", "bad":true, "user":"AAB", "query":"AC", "line":10, "node":"AB"} +{"world":"CAC", "user":"AB", "query":"ACA", "indexed":true, "line":11, "space":"CB"} +{"line":12, "pos":72, "abstract":"BBA", "space":"AAC"} +{} +{"world":"CC", "query":"AA", "line":14, "disabled":false, "date":"CAC", "coauthors":"AB"} +{"org":68, "title":"BBB", "query":"BAC", "line":15, "public":false} +{"org":73, "user":"AA", "indexed":true, "line":16, "date":"CCC", "public":true, "coauthors":"AB"} +{"indexed":false, "line":17} +{"state":23, "auth":"BCC", "org":38, "status":28, "line":18, "disabled":false, "abstract":"CB"} +{"state":99, "auth":"CA", "indexed":true, "line":19, "date":"BA"} +{"wait":"CBA", "user":"BBA", "indexed":true, "line":20, "disabled":false, "abstract":"BA", "date":"ABA"} +{"org":10, "query":"AC", "indexed":false, "line":21, "disabled":true, "abstract":"CA", "pos":44} +{"state":65, "title":"AC", "user":"AAC", "cleaned":true, "status":93, "line":22, "abstract":"ABC", "node":"CCC"} +{"subtitle":"AC", "user":"CCC", "line":23} +{"state":67, "world":"ACB", "bad":true, "user":"CB", "line":24, "disabled":true} +{} \ No newline at end of file diff --git a/contrib/try_convert/data/tt_macaddr.data b/contrib/try_convert/data/tt_macaddr.data new file mode 100644 index 00000000000..4a05521feab --- /dev/null +++ b/contrib/try_convert/data/tt_macaddr.data @@ -0,0 +1,12 @@ +08:00:2b:01:02:03 +08-00-2b-01-02-03 +08002b:010203 +08002b-010203 +0800.2b01.0203 +08002b010203 +08:00:2b:01:02:03 +08-00-2b-01-02-03 +08002b:010203 +08002b-010203 +0800.2b01.0203 +08002b010203 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_money.data b/contrib/try_convert/data/tt_money.data new file mode 100644 index 00000000000..3ec7528e04c --- /dev/null +++ b/contrib/try_convert/data/tt_money.data @@ -0,0 +1,16 @@ +10 +0.01 +10.03 +$9 +$1,000,000,000,000,000.00 +0 +$555555555 +1,656,343 +10 +0.01 +10.03 +$9 +$1,000,000,000,000,000.00 +0 +$555555555 +1,656,343 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_numeric.data b/contrib/try_convert/data/tt_numeric.data new file mode 100644 index 00000000000..0651147684a --- /dev/null +++ b/contrib/try_convert/data/tt_numeric.data @@ -0,0 +1,30 @@ +5.49803593494943 +2.65056628940059 +87.2433041085257 +42.3137940200886 +211.798205442082 +539.296088779458 +7299.31069089976 +2011.51063389695 +31171.6291300894 +99514.9356660894 +649878.057639453 +438100.083914504 +5175758.4103559 +1210041.95868265 +22469733.7031557 +33808556.2147455 +588308718.457233 +230114732.596577 +2202173844.51559 +709930860.090325 +63110295727.0098 +22894178381.1154 +905420013006.127 +859635400253.746 +708573498886.534 +2380046343689.95 +66897777829628.1 +21423680737043.2 +132311848725025 +935514240580671 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_oid.data b/contrib/try_convert/data/tt_oid.data new file mode 100644 index 00000000000..1d1d3048863 --- /dev/null +++ b/contrib/try_convert/data/tt_oid.data @@ -0,0 +1,13 @@ +232 +2908 +1069 +20 +21 +24 +2950 +701 +18 +19 +114 +1247 +1259 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_point.data b/contrib/try_convert/data/tt_point.data new file mode 100644 index 00000000000..611184758d0 --- /dev/null +++ b/contrib/try_convert/data/tt_point.data @@ -0,0 +1,16 @@ +(1, 2) +(-100000, 0) +21, 9999999999 +0.011111, 9 +962856498.3423, -243.24 +3321, 123 +23,12 +1321 , 132 +216 ,345354 +( 21, 9999999999 ) +( 0.011111, 9 ) +( 962856498.3423, -243.24 ) +(3321, 123 ) +( 23,12) +(1321 , 132 ) +( 216 ,345354) \ No newline at end of file diff --git a/contrib/try_convert/data/tt_regclass.data b/contrib/try_convert/data/tt_regclass.data new file mode 100644 index 00000000000..df7234772a6 --- /dev/null +++ b/contrib/try_convert/data/tt_regclass.data @@ -0,0 +1,12 @@ +pg_type +pg_proc +pg_class +pg_attribute +pg_user +pg_statistic +pg_class_oid_index +pg_views +pg_timezone_names +pg_stat_database +pg_tables +pg_roles \ No newline at end of file diff --git a/contrib/try_convert/data/tt_regproc.data b/contrib/try_convert/data/tt_regproc.data new file mode 100644 index 00000000000..c1fdfdc54fd --- /dev/null +++ b/contrib/try_convert/data/tt_regproc.data @@ -0,0 +1,11 @@ +textin +int4lt +float8abs +232 +2908 +now +pg_stat_get_activity +pg_table_size +1069 +inet_client_addr +inet_server_addr \ No newline at end of file diff --git a/contrib/try_convert/data/tt_regtype.data b/contrib/try_convert/data/tt_regtype.data new file mode 100644 index 00000000000..187bba10479 --- /dev/null +++ b/contrib/try_convert/data/tt_regtype.data @@ -0,0 +1,30 @@ +bool + bytea + char + name + int8 + int2 + int2vector + int4 + regproc + text + oid + tid + xid + cid + oidvector + pg_type + pg_attribute + pg_proc + pg_class + json + xml + _xml + pg_node_tree + _json + complex + _complex + smgr + point + lseg + path \ No newline at end of file diff --git a/contrib/try_convert/data/tt_reltime.data b/contrib/try_convert/data/tt_reltime.data new file mode 100644 index 00000000000..3f44dbbc925 --- /dev/null +++ b/contrib/try_convert/data/tt_reltime.data @@ -0,0 +1,12 @@ +16736 days, 13:35:45 +1506 days, 12:52:07 +18099 days, 20:19:33 +19898 days, 20:02:00 +6180 days, 9:16:50 +4727 days, 16:01:59 +3484 days, 5:59:54 +4560 days, 15:52:43 +11498 days, 17:23:38 +1873 days, 16:50:17 +13678 days, 10:50:09 +1257 days, 9:06:52 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_text.data b/contrib/try_convert/data/tt_text.data new file mode 100644 index 00000000000..21c17124dfc --- /dev/null +++ b/contrib/try_convert/data/tt_text.data @@ -0,0 +1,70 @@ +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. + +std::to_string + C++ Strings library std::basic_string +Defined in header +std::string to_string( int value ); +std::string to_string( long value ); +std::string to_string( long long value ); +std::string to_string( unsigned value ); +std::string to_string( unsigned long value ); +std::string to_string( unsigned long long value ); +std::string to_string( float value ); +std::string to_string( double value ); +std::string to_string( long double value ); +Converts a numeric value to std::string. + +Let buf be an internal to the conversion functions buffer, sufficiently large to contain the result of conversion. + +1) Converts a signed integer to a string as if by std::sprintf(buf, "%d", value). +2) Converts a signed integer to a string as if by std::sprintf(buf, "%ld", value). +3) Converts a signed integer to a string as if by std::sprintf(buf, "%lld", value). +4) Converts an unsigned integer to a string as if by std::sprintf(buf, "%u", value). +5) Converts an unsigned integer to a string as if by std::sprintf(buf, "%lu", value). +6) Converts an unsigned integer to a string as if by std::sprintf(buf, "%llu", value). +7,8) Converts a floating point value to a string as if by std::sprintf(buf, "%f", value). +9) Converts a floating point value to a string as if by std::sprintf(buf, "%Lf", value). +(until C++26) +1-9) Converts a numeric value to a string as if by std::format("{}", value). +(since C++26) +Parameters +Return value +A string holding the converted value. + +Exceptions +May throw std::bad_alloc from the std::string constructor. + +Notes +With floating point types std::to_string may yield unexpected results as the number of significant digits in the returned string can be zero, see the example. +The return value may differ significantly from what std::cout prints by default, see the example. +std::to_string relies on the current C locale for formatting purposes, and therefore concurrent calls to std::to_string from multiple threads may result in partial serialization of calls. +The results of overloads for integer types do not rely on the current C locale, and thus implementations generally avoid access to the current C locale in these overloads for both correctness and performance. However, such avoidance is not guaranteed by the standard. +(until C++26) +C++17 provides std::to_chars as a higher-performance locale-independent alternative. \ No newline at end of file diff --git a/contrib/try_convert/data/tt_time.data b/contrib/try_convert/data/tt_time.data new file mode 100644 index 00000000000..92ba52ea639 --- /dev/null +++ b/contrib/try_convert/data/tt_time.data @@ -0,0 +1,12 @@ +13:35:45 +21:52:07 +16:19:33 +20:02:00 +11:16:50 +01:01:59 +14:59:54 +10:52:43 +19:23:38 +01:50:17 +09:50:09 +09:06:52 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_timestamp.data b/contrib/try_convert/data/tt_timestamp.data new file mode 100644 index 00000000000..5cf22efbee9 --- /dev/null +++ b/contrib/try_convert/data/tt_timestamp.data @@ -0,0 +1,12 @@ +2015-10-28 16:35:45 +1974-02-15 15:52:07 +2019-07-22 23:19:33 +2024-06-24 23:02:00 +1986-12-03 12:16:50 +1982-12-11 19:01:59 +1979-07-17 08:59:54 +1982-06-27 18:52:43 +2001-06-25 20:23:38 +1975-02-17 19:50:17 +2007-06-14 13:50:09 +1973-06-11 12:06:52 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_timestamptz.data b/contrib/try_convert/data/tt_timestamptz.data new file mode 100644 index 00000000000..bdb863b79f7 --- /dev/null +++ b/contrib/try_convert/data/tt_timestamptz.data @@ -0,0 +1,12 @@ +2015-10-28 13:35:45+00:00 +1974-02-15 21:52:07+09:00 +2019-07-22 16:19:33-04:00 +2024-06-24 20:02:00+00:00 +1986-12-03 11:16:50+02:00 +1982-12-12 01:01:59+09:00 +1979-07-17 14:59:54+09:00 +1982-06-27 10:52:43-04:00 +2001-06-25 19:23:38+03:00 +1975-02-18 01:50:17+09:00 +2007-06-14 09:50:09+00:00 +1973-06-11 09:06:52+00:00 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_timetz.data b/contrib/try_convert/data/tt_timetz.data new file mode 100644 index 00000000000..cee4c87ffb3 --- /dev/null +++ b/contrib/try_convert/data/tt_timetz.data @@ -0,0 +1,12 @@ +13:35:45 UTC +21:52:07 JST +16:19:33 EDT +20:02:00 UTC +11:16:50 EET +01:01:59 JST +14:59:54 JST +10:52:43 EDT +19:23:38 EEST +01:50:17 JST +09:50:09 UTC +09:06:52 UTC \ No newline at end of file diff --git a/contrib/try_convert/data/tt_uuid.data b/contrib/try_convert/data/tt_uuid.data new file mode 100644 index 00000000000..1ed8f566e0f --- /dev/null +++ b/contrib/try_convert/data/tt_uuid.data @@ -0,0 +1,12 @@ +a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11 +{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11} +a0eebc999c0b4ef8bb6d6bb9bd380a11 +a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11 +{a0eebc99-9c0b4ef8-bb6d6bb9-bd380a11} +a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11 +{a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11} +a0eebc999c0b4ef8bb6d6bb9bd380a11 +a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11 +{a0eebc99-9c0b4ef8-bb6d6bb9-bd380a11} \ No newline at end of file diff --git a/contrib/try_convert/data/tt_varbit.data b/contrib/try_convert/data/tt_varbit.data new file mode 100644 index 00000000000..f6724da39ab --- /dev/null +++ b/contrib/try_convert/data/tt_varbit.data @@ -0,0 +1,11 @@ +1 +0 +1010101 +11001010 +1011011101111 +01001 +1111 +00000000000000000000000 +1111111111111111111 +0101001010101 +11101001000100 \ No newline at end of file diff --git a/contrib/try_convert/data/tt_varchar.data b/contrib/try_convert/data/tt_varchar.data new file mode 100644 index 00000000000..21c17124dfc --- /dev/null +++ b/contrib/try_convert/data/tt_varchar.data @@ -0,0 +1,70 @@ +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. + +std::to_string + C++ Strings library std::basic_string +Defined in header +std::string to_string( int value ); +std::string to_string( long value ); +std::string to_string( long long value ); +std::string to_string( unsigned value ); +std::string to_string( unsigned long value ); +std::string to_string( unsigned long long value ); +std::string to_string( float value ); +std::string to_string( double value ); +std::string to_string( long double value ); +Converts a numeric value to std::string. + +Let buf be an internal to the conversion functions buffer, sufficiently large to contain the result of conversion. + +1) Converts a signed integer to a string as if by std::sprintf(buf, "%d", value). +2) Converts a signed integer to a string as if by std::sprintf(buf, "%ld", value). +3) Converts a signed integer to a string as if by std::sprintf(buf, "%lld", value). +4) Converts an unsigned integer to a string as if by std::sprintf(buf, "%u", value). +5) Converts an unsigned integer to a string as if by std::sprintf(buf, "%lu", value). +6) Converts an unsigned integer to a string as if by std::sprintf(buf, "%llu", value). +7,8) Converts a floating point value to a string as if by std::sprintf(buf, "%f", value). +9) Converts a floating point value to a string as if by std::sprintf(buf, "%Lf", value). +(until C++26) +1-9) Converts a numeric value to a string as if by std::format("{}", value). +(since C++26) +Parameters +Return value +A string holding the converted value. + +Exceptions +May throw std::bad_alloc from the std::string constructor. + +Notes +With floating point types std::to_string may yield unexpected results as the number of significant digits in the returned string can be zero, see the example. +The return value may differ significantly from what std::cout prints by default, see the example. +std::to_string relies on the current C locale for formatting purposes, and therefore concurrent calls to std::to_string from multiple threads may result in partial serialization of calls. +The results of overloads for integer types do not rely on the current C locale, and thus implementations generally avoid access to the current C locale in these overloads for both correctness and performance. However, such avoidance is not guaranteed by the standard. +(until C++26) +C++17 provides std::to_chars as a higher-performance locale-independent alternative. \ No newline at end of file diff --git a/contrib/try_convert/data/tt_xml.data b/contrib/try_convert/data/tt_xml.data new file mode 100644 index 00000000000..389588a6525 --- /dev/null +++ b/contrib/try_convert/data/tt_xml.data @@ -0,0 +1,10 @@ + + on main_window 500 500 250 250 center text1 250 100 center sun1.opacity = (sun1.opacity / 100) * 90; + A Song of Ice and Fire George R. R. Martin English Epic fantasy + Rick Grimes 35 Maths Male Daryl Dixon 33 Science Male Maggie 36 Arts Female +
Adobe SVG Viewer
Open Open New Zoom In Zoom Out Original View Quality Pause Mute Find... Find Again Copy Copy Again Copy SVG View SVG View Source Save As Help About Adobe CVG Viewer... + + on main_window 500 500 250 250 center text1 250 100 center sun1.opacity = (sun1.opacity / 100) * 90; + A Song of Ice and Fire George R. R. Martin English Epic fantasy + Rick Grimes 35 Maths Male Daryl Dixon 33 Science Male Maggie 36 Arts Female +
Adobe SVG Viewer
Open Open New Zoom In Zoom Out Original View Quality Pause Mute Find... Find Again Copy Copy Again Copy SVG View SVG View Source Save As Help About Adobe CVG Viewer... \ No newline at end of file diff --git a/contrib/try_convert/scripts/check_test.py b/contrib/try_convert/scripts/check_test.py new file mode 100644 index 00000000000..52e66ed158e --- /dev/null +++ b/contrib/try_convert/scripts/check_test.py @@ -0,0 +1,87 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re + +regression_path = './regression.diffs' + +f = open(regression_path) +lines = f.read().split('\n') + +needed_types = [ + 'bool', + + 'int2', + 'int4', + 'int8', + 'float4', + 'float8', + 'numeric', + + 'date', + 'time', + 'timestamp', + 'timetz', + 'timestamptz', + 'interval' + + 'regproc', + 'pg_catalog', + 'regclass', + 'regtype', + + 'value_day', + 'oid', + 'jsonb', + 'json', + + # 'text', + # 'bpchar', + # 'varchar', + # 'char' +] + +failed_tests = {} +c = 0 + +for i in range(1, len(lines)): + preline = lines[i-1] + line = lines[i] + if len(line) > 0 and len(preline) > 0 and (line[0] == '-' or line[0] == '+') and (preline[0] != '-' and preline[0] != '+'): + words = re.split('::|\*|;|\n| |\(|\)|,|\.|\".*\"|\'.*\'|<.*>', preline) + ans = [] + is_prining = False + for word in words: + if word not in ['select', 'from', 'count', + 'try_convert', 'try_convert_by_sql', 'try_convert_by_sql_text', 'try_convert_by_sql_with_len_out', + 'NULL', 'v', 'v1', 'v2', 'where', 'is', 'not', 'distinct', 'as', 't', '']: + ans += [word] + for w in word.split('_'): + if w in needed_types: + is_prining = True + if is_prining: + failed_tests[' '.join(ans)] = True + # print(ans, line) + c += 1 + +for ft in failed_tests: + print(ft) + +print(f'Summary: {c}') + \ No newline at end of file diff --git a/contrib/try_convert/scripts/error_safe_check.py b/contrib/try_convert/scripts/error_safe_check.py new file mode 100644 index 00000000000..2852f5eff3e --- /dev/null +++ b/contrib/try_convert/scripts/error_safe_check.py @@ -0,0 +1,143 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re + +import os, glob + +p_any = '(?:\S| |\t|\n+\t|\n+ |\n+#|\n+/|\n+\$)+?' +p_spaces = '\s*' + +def create_pattern(funcCall): + return '\n(\w+)\s+?(\w+)' + p_spaces + '\((' + p_any + ')\)' + p_spaces + '(\n\{' + '(' + p_any + ')' + f'({funcCall})' + '\s*' + '(' + p_any + ')' + '\n+\})' + +pattern = create_pattern('ereturn') + + +source_filenames = ['time.c', 'date.c', 'timedate.c', 'int.c', 'float.c', 'bool.c', 'char.c', 'formatting.c', 'json.c', 'jsonb.c', 'nabstime.c', 'numeric.c', 'timestamp.c', 'network.c'] + + +# verify all convert and in&outs(if in converts) are 'soft'-error handling +# safe error handle == sub-calls are not 'ereport' (replaced by 'ereturn') + + +boolin = ''' + const char *in_str = PG_GETARG_CSTRING(0); + const char *str; + size_t len; + bool result; + + /* + * Skip leading and trailing whitespace + */ + str = in_str; + while (isspace((unsigned char) *str)) + str++; + + len = strlen(str); + while (len > 0 && isspace((unsigned char) str[len - 1])) + len--; + + if (parse_bool_with_len(str, len, &result)) + PG_RETURN_BOOL(result); + + ereturn(fcinfo->context, 0, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type boolean: \"%s\"", in_str))); + + /* not reached */ + PG_RETURN_BOOL(false); +''' + + +def find_functions_with_call(functions): + ddd = '|'.join(functions + ['ereturn']) + call_pattern = f'(?:{ddd})' + function_with_call_pattern = create_pattern(call_pattern) + + caller_functions = [] + caller_pg_functions = [] + + unadapted_functions = {} + unadapted_calls = [] + functions_with_unadapted_calls = {} + + for root, subdirs, files in os.walk('../..'): + for filename in files: + file_path = os.path.join(root, filename) + # if filename[-2:] == '.c': + if filename in source_filenames: + # print(file_path) + with open(file_path, 'r') as f: + content = f.read() + # content = 'lol () { print(\'sds\') ereturn(wwe); }' + matches = re.findall(function_with_call_pattern, content) + if len(matches) > 0: + for m in matches: + # print('!!!!!!!!!!', file_path, m[0], m[1], m[2], m[3], '$$$$$$$$', m[4]) + caller_functions += [m[1]] + if m[2] == 'PG_FUNCTION_ARGS': + caller_pg_functions += [m[1]] + + if (m[0] != 'bool' or re.search('escontext', m[2]) is None) and m[2] != 'PG_FUNCTION_ARGS': + unadapted_functions[m[1]] = True + + if m[5] != 'ereturn': + func_call = m[4][-20:] + m[5] + m[6][:20] + safe_call_pattern = f'if{p_spaces}\(!{m[5]}\({p_any}\)\)' + safe_call_void_pattern = f'\(void\){p_spaces}{m[5]}\({p_any}\)\)' + direct_safe_call_pattern = f'DirectFunctionCall1Safe\({m[5]}' + if re.search(safe_call_pattern, func_call) is None and \ + re.search(safe_call_void_pattern, func_call) is None and \ + re.search(direct_safe_call_pattern, func_call) is None: + unadapted_calls += [m[5]] + + if m[1] in functions_with_unadapted_calls: + functions_with_unadapted_calls[m[1]] += [m[5]] + else: + functions_with_unadapted_calls[m[1]] = [m[5]] + + return caller_functions, caller_pg_functions, unadapted_functions, unadapted_calls, functions_with_unadapted_calls + +caller_functions = [] + +# print(re.findall('!' + p_any + '\n*!', f'!{boolin}!')) + +while True: + new_caller_functions, new_caller_pg_functions, unadapted_functions, unadapted_calls, functions_with_unadapted_calls = find_functions_with_call(caller_functions) + + if len(caller_functions) == len(new_caller_functions): + break + + # for c in sorted(new_caller_functions): + # if c not in caller_functions: + # print(c) + + for f in unadapted_functions: + print(f) + + caller_functions = new_caller_functions + + print(len(caller_functions), len(unadapted_functions), len(unadapted_calls), len(functions_with_unadapted_calls)) + +# replase res = func(intput) -> if (!func(input, &res, escontext)) return false; +# define safe_call(functionCall, input...) if (!functionCall(input)) return false; + +for f in sorted(functions_with_unadapted_calls): + print(f, functions_with_unadapted_calls[f]) \ No newline at end of file diff --git a/contrib/try_convert/scripts/find_calls.py b/contrib/try_convert/scripts/find_calls.py new file mode 100644 index 00000000000..328307bd911 --- /dev/null +++ b/contrib/try_convert/scripts/find_calls.py @@ -0,0 +1,116 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +import os, glob + +from general import source_filenames + +p_any = '(?:\S| |\t|\n+\t|\n+ |\n+#|\n+/|\n+\$)+?' +p_spaces = '\s*' +p_space = '\s+' + + +def remove_comments(body): + body = re.sub('".*?"', '""', body) + body = re.sub('/\*[\s\S]*?\*/', '/* */', body) + return body + +def find_functions(func_names, text): + + func_names_pattern_list = '|'.join(func_names) + func_names_pattern = f'((?:{func_names_pattern_list}))' + + func_pattern = '\n(\w+)\s+?' + func_names_pattern + p_spaces + '\((' + p_any + ')\)' + p_spaces + '(\n\{' + '(' + p_any + ')' + '\n+\})' + + funcs = [] + + for m in re.findall(func_pattern, text): + # print(m[1]) + funcs += [(m[1], m[0], m[2], m[3])] + + + return funcs + +def find_safe_functions(text): + + func_pattern = '\n((?:\w+\s+)+)(\w+)' + p_spaces + '\((' + p_any + 'Node' + p_spaces + '\*' + p_spaces + 'escontext' + ')\)' + p_spaces + '(\n\{' + '(' + p_any + ')' + '\n+\})' + + funcs = [] + + for m in re.findall(func_pattern, text): + # print(m[1]) + funcs += [(m[1], m[0], m[2], m[3])] + + + return funcs + +def create_pattern(funcCall): + return '\n((?:\w+\s+)+)(\w+)' + p_spaces + '\((' + p_any + ')\)' + p_spaces + '(\n\{' + '(' + p_any + ')' + '\W' + f'({funcCall})' + '\W' + '(' + p_any + ')' + '\n+\})' + + +def find_functions_with_call(functions): + ddd = '|'.join(functions) + call_pattern = f'(?:{ddd})' + function_with_call_pattern = create_pattern(call_pattern) + + caller_functions = [] + + for root, subdirs, files in os.walk('../..'): + for filename in files: + file_path = os.path.join(root, filename) + # if filename[-2:] == '.c': + if filename in source_filenames: + # print(file_path) + with open(file_path, 'r') as f: + content = remove_comments(f.read()) + # content = 'lol () { print(\'sds\') ereturn(wwe); }' + matches = re.findall(function_with_call_pattern, content) + if len(matches) > 0: + for m in matches: + # print('!!!!!!!!!!', file_path, m[0], m[1], m[2], m[3], '$$$$$$$$', m[4]) + # if m[1] == 'DecodeNumberField': + # print(m[1], m[4], m[5]) + if m[1] not in ['if']: + caller_functions += [m[1]] + + return caller_functions + + +def get_all_functions_with(function_call): + + + caller_functions = {function_call} + + # print(re.findall('!' + p_any + '\n*!', f'!{boolin}!')) + + while True: + new_caller_functions = find_functions_with_call(list(caller_functions)) + + l = len(caller_functions) + + for cf in new_caller_functions: + caller_functions.add(cf) + + if l == len(caller_functions): + break + + # print(len(caller_functions)) + + return list(caller_functions) \ No newline at end of file diff --git a/contrib/try_convert/scripts/find_casts.py b/contrib/try_convert/scripts/find_casts.py new file mode 100644 index 00000000000..23e20c52f41 --- /dev/null +++ b/contrib/try_convert/scripts/find_casts.py @@ -0,0 +1,248 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re +import os, glob + + +top_srcdir = '../..' + +pg_type_path = f'{top_srcdir}/src/include/catalog/pg_type.dat' +pg_cast_path = f'{top_srcdir}/src/include/catalog/pg_cast.dat' +pg_proc_path = f'{top_srcdir}/src/include/catalog/pg_proc.dat' + + +def get_pg_proc(): + f = open(pg_proc_path) + content = f.read() + + # { oid => '42', descr => 'I/O', + # proname => 'int4in', prorettype => 'int4', proargtypes => 'cstring', + # prosrc => 'int4in' }, + # func_pattern = r'DATA\(insert OID =\s+(\w*)\s+\(.*?(\w*) _null_ _null_ _null_ n?\s?a?\s?\)\);'; + func_pattern = r"\{ oid => '(\w*)',[\s\S]*?prosrc => '(\w*)'[\s\S]*?\}"; + + func_id_name = {} + + # func_id_name['0'] = 'via I/O' + + for (id, name) in re.findall(func_pattern, content): + func_id_name[id] = name + + print('func_id_name', len(func_id_name)) + + return func_id_name + + +def get_pg_type(): + f = open(pg_type_path) + content = f.read() + + # { oid => '23', array_type_oid => '1007', + # descr => '-2 billion to 2 billion integer, 4-byte storage', + # typname => 'int4', typlen => '4', typbyval => 't', typcategory => 'N', + # typinput => 'int4in', typoutput => 'int4out', typreceive => 'int4recv', + # typsend => 'int4send', typalign => 'i' }, + type_pattern = r"\{ oid => '(\w*)',[\s\S]*?typname => '(\w*)'[\s\S]*?typinput => '(\w*)'[\s\S]*?typoutput => '(\w*)'[\s\S]*?\}"; + + type_name_id = {} + type_id_name = {} + type_io_funcs = {} + + for t in re.findall(type_pattern, content): + id = t[0] + name = t[1] + infunc = t[2] + outfunc = t[3] + + type_io_funcs[name] = [infunc, outfunc] + # print(len(t), t[3]) + if name != '' and name[0] != '_': + id = int(id) + type_id_name[id] = name + type_name_id[name] = id + + print('type_name_id', len(type_name_id)) + + return type_name_id, type_id_name, type_io_funcs + + +def get_pg_cast(type_id_name, func_id_name): + f = open(pg_cast_path) + content = f.read() + +# { castsource => 'int8', casttarget => 'int4', castfunc => 'int4(int8)', +# castcontext => 'a', castmethod => 'f' }, + # cast_pattern = r'DATA\(insert \([\s]*(\d+)[\s]+(\d+)[\s]+(\d+)[\s]+(.)[\s]+(.)'; + cast_pattern = r"\{ castsource => '(\w+)', casttarget => '(\w+)', castfunc => '(\w+)',\s*castcontext => '(\w+)', castmethod => '(\w+)' \}"; + + casts = [] + + for (source, target, funcid, _, meth) in re.findall(cast_pattern, content): + # if int(source) not in type_id_name or int(target) not in type_id_name: + # continue + + way = '______unknown' + + if meth == 'f': + if funcid not in func_id_name: + way = '______unknown_funcid' + else: + way = func_id_name[funcid] + if way == '': + way = '______unknown_funcid' + + elif meth == 'i': + way = 'WITH INOUT' + elif meth == 'b': + way = 'WITHOUT FUNCTION' + + + + + casts += [(source, target, way)] + # print(type_id_name[int(source)], ' -> ', type_id_name[int(target)], ' via ', meth, f'({funcid} - {func_id_name[funcid]}) ', f'{source}-{target}') + + print('casts', len(casts)) + + return casts + + +### GET FROM TEXT (EXTENSIONS) + +from general import supported_extensions + +def get_extensions(): + + create_casts = [] + create_functions = [] + + for extension in supported_extensions: + for root, subdirs, files in os.walk(f'../{extension}'): + for filename in files: + file_path = os.path.join(root, filename) + if filename[-4:] == '.sql': + # print(file_path) + with open(file_path, 'r') as f: + content = f.read() + + create_casts += find_create_casts_in_text(content) + + create_functions += list(find_create_function_in_text(content).items()) + + return create_casts, dict(create_functions) + +def find_create_casts_in_text(text): + create_cast_pattern = 'CREATE CAST\s*\((\w+) AS (\w+)\)\s*([\w\s\(\)]+);' + create_casts = [] + + for target, source, f in re.findall(create_cast_pattern, text): + # print(target, source, f) + if re.match('WITH INOUT', f) is not None: + create_casts += [(target, source, 'WITH INOUT')] + if re.match('WITHOUT FUNCTION', f) is not None: + create_casts += [(target, source, 'WITHOUT FUNCTION')] + + m = re.match('WITH FUNCTION ([\w\(\)]+)', f) + if m is not None: + # print(m[1]) + create_casts += [(target, source, m[1])] + + return create_casts + +p_space = '\s+' + +def find_create_function_in_text(text): + create_function_pattern = 'CREATE FUNCTION' + p_space + '(\w+)\([\w\s,]+\)' + p_space + 'RETURNS' + p_space + '\w+' + p_space + 'AS' + p_space + '([\',\w\s]+)' + p_space + 'LANGUAGE[\w\s,]+;' + create_functions = {} + + for sql_name, c_obj in re.findall(create_function_pattern, text): + # print(target, source, f) + + m = re.fullmatch("\'(\w+)\'", c_obj) + if m is not None: + c_name = m[1] + create_functions[sql_name] = c_name + continue + + m = re.fullmatch("\'MODULE_PATHNAME\',\s+\'(\w+)\'", c_obj) + if m is not None: + c_name = m[1] + create_functions[sql_name] = c_name + + + return create_functions + + + +EXAMPLE_CRETATE_CAST_EXTENSION = ''' +# CITEXT + +CREATE CAST (citext AS text) WITHOUT FUNCTION AS IMPLICIT; +CREATE CAST (citext AS varchar) WITHOUT FUNCTION AS IMPLICIT; +CREATE CAST (citext AS bpchar) WITHOUT FUNCTION AS ASSIGNMENT; +CREATE CAST (text AS citext) WITHOUT FUNCTION AS ASSIGNMENT; +CREATE CAST (varchar AS citext) WITHOUT FUNCTION AS ASSIGNMENT; +CREATE CAST (bpchar AS citext) WITH FUNCTION citext(bpchar) AS ASSIGNMENT; +CREATE CAST (boolean AS citext) WITH FUNCTION citext(boolean) AS ASSIGNMENT; +CREATE CAST (inet AS citext) WITH FUNCTION citext(inet) AS ASSIGNMENT; + +CREATE FUNCTION citext(bpchar) +RETURNS citext +AS 'rtrim1' +LANGUAGE internal IMMUTABLE STRICT; + +CREATE FUNCTION citext(boolean) +RETURNS citext +AS 'booltext' +LANGUAGE internal IMMUTABLE STRICT; + +CREATE FUNCTION citext(inet) +RETURNS citext +AS 'network_show' +LANGUAGE internal IMMUTABLE STRICT; + + + +# HSTORE + +CREATE CAST (text[] AS hstore) + WITH FUNCTION hstore(text[]); + +CREATE CAST (hstore AS json) + WITH FUNCTION hstore_to_json(hstore); + +CREATE CAST (hstore AS jsonb) + WITH FUNCTION hstore_to_jsonb(hstore); + +CREATE FUNCTION hstore(text[]) +RETURNS hstore +AS 'MODULE_PATHNAME', 'hstore_from_array' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION hstore_to_json(hstore) +RETURNS json +AS 'MODULE_PATHNAME', 'hstore_to_json' +LANGUAGE C IMMUTABLE STRICT; + +CREATE FUNCTION hstore_to_jsonb(hstore) +RETURNS jsonb +AS 'MODULE_PATHNAME', 'hstore_to_jsonb' +LANGUAGE C IMMUTABLE STRICT; +''' \ No newline at end of file diff --git a/contrib/try_convert/scripts/find_ereturns.py b/contrib/try_convert/scripts/find_ereturns.py new file mode 100644 index 00000000000..d26d70bfafe --- /dev/null +++ b/contrib/try_convert/scripts/find_ereturns.py @@ -0,0 +1,67 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re + +import os, glob + +from general import source_filenames + +p_any = '(?:\S| |\t|\n+\t|\n+ |\n+#|\n+/|\n+\$)+?' +p_anys = '(?:\S| |\t|\n+\t|\n+ |\n+#|\n+/|\n+\$)*?' +p_spaces = '\s*' +p_space = '\s+' + +ereturn_pattern = 'ereturn\(' + p_any + '\);' + +ereturns = {} + +for root, subdirs, files in os.walk('../..'): + for filename in files: + file_path = os.path.join(root, filename) + # if filename[-2:] == '.c': + if filename in source_filenames: + # print(file_path) + with open(file_path, 'r') as f: + content = f.read() + # content = 'lol () { print(\'sds\') ereturn(wwe); }' + matches = re.findall(ereturn_pattern, content) + + for m in matches: + + def get_field(field, m): + errfield = re.search(f'{field}\("('+ p_anys + '"' + p_anys + ')[\)\n]', m) + if errfield is not None: + errfield = errfield[1] + + return errfield + + errmsg = get_field('errmsg', m) + errdetail = get_field('errdetail', m) + errhint = get_field('errhint', m) + + desc = (errmsg, errdetail, errhint) + + ereturns[filename] = (ereturns[filename] if filename in ereturns else []) + [desc] + + +for filename in sorted(ereturns): + print(filename) + for ereturn in ereturns[filename]: + print(' ', ereturn) \ No newline at end of file diff --git a/contrib/try_convert/scripts/general.py b/contrib/try_convert/scripts/general.py new file mode 100644 index 00000000000..e0677e3e0d7 --- /dev/null +++ b/contrib/try_convert/scripts/general.py @@ -0,0 +1,136 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source_filenames = [ + 'time.c', 'date.c', 'datetime.c', 'timestamp.c', 'nabstime.c', 'formatting.c', + 'int.c', 'int8.c', 'float.c', 'float.c', 'bool.c', 'numeric.c', 'numutils.c', + 'format_type.c', + # 'varbit.c', + 'char.c', 'varchar.c', + 'json.c', 'jsonb.c', 'xml.c', + 'network.c', + # 'cash.c', + 'reg_proc.c', + 'postgres.c', + 'stringinfo.c', + ] + [ + 'citext.c', 'oracle_compat.c', + 'hstore_io.c', + ] + +supported_types = [ + 'int8', # NUMBERS + 'int4', + 'int2', + 'float8', + 'float4', + 'numeric', + # 'complex', + + 'bool', + + # 'bit', # BITSTRING + # 'varbit', + + 'date', # TIME + 'time', + 'timetz', + 'timestamp', + 'timestamptz', + 'interval', + + # 'point', # GEOMENTY + # 'circle', + # 'line', + # 'lseg', + # 'path', + # 'box', + # 'polygon', + + # 'cidr', # IP + # 'inet', + # 'macaddr', + + 'json', # OBJ + 'jsonb', + # 'xml', + + # 'bytea', + + 'char', # STRINGS + # 'bpchar', + 'varchar', + 'text', + + # 'money', + # # 'pg_lsn', + # # 'tsquery', + # # 'tsvector', + # # 'txid_snapshot', + # 'uuid', + + # 'regtype', # SYSTEM + # 'regproc', + # 'regclass', + # 'oid', +] + [ + 'citext', + 'hstore', +] + +supported_extensions = [ + 'citext', + 'hstore', +] + + + +string_types = [ + 'text', + 'citext', + 'char', + # 'bpchar', + 'varchar', +] + +typmod_types = [ + 'bit', + 'varbit', + 'char', + 'varchar', + # 'bpchar', +] + +typmod_lens = [ + None, 1, 5, 10, 20 +] + + + +uncomparable_types = [ + 'json', + 'xml', + 'point', +] + +has_corrupt_data = [ + 'time', 'timetz', 'timestamp', 'timestamptz', 'date', + 'json', + 'int2', 'int4', 'int8', 'float4', 'float8', 'numeric', +] \ No newline at end of file diff --git a/contrib/try_convert/scripts/generate_data.py b/contrib/try_convert/scripts/generate_data.py new file mode 100644 index 00000000000..b4aa4aec941 --- /dev/null +++ b/contrib/try_convert/scripts/generate_data.py @@ -0,0 +1,119 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import random +import datetime +import time +import pytz + +random.seed(42) + +### NUMBERS + +numbers = { + 'int2' : ((-32768, 32767), False), + 'int4' : ((-2147483648, 2147483647), False), + 'int8' : ((-9223372036854775808, 9223372036854775807), False), + 'float4' : ((10**6, 10**6), True), + 'float8' : ((10**15, 10**15), True), + 'numeric' : ((10**20, 10**20), True), +} + + +def save_datafile(t, data): + filename = f'data/tt_{t}.data' + file = open(filename, 'w') + file.write('\n'.join([str(d) for d in data])) + + return filename + + +for number_type in numbers: + + type_range = numbers[number_type][0] + is_float = numbers[number_type][1] + + mn = type_range[0] + mx = type_range[1] + + nln = 0 + ln = len(str(type_range[1]))-1 + + table_name = f'tt_{number_type}' + + values = [] + + for c in range(nln, ln): + rep = 20 // ln + 1 + for _ in range(rep): + n = random.random() + if c >= 0: + n *= (10 ** (c+1)) + else: + n /= (10 ** (-c)) + if not is_float: + n = int(n) + + values += [n] + + filename = save_datafile(number_type, values) + +### TIMES + +MINTIME = datetime.datetime.fromtimestamp(0) +MAXTIME = datetime.datetime(2024,12,2,10,39,59) +mintime_ts = int(time.mktime(MINTIME.timetuple())) +maxtime_ts = int(time.mktime(MAXTIME.timetuple())) + +timestamp_values = [] +timestamptz_values = [] +time_values = [] +timetz_values = [] +date_values = [] +interval_values = [] + +timezones = [pytz.timezone("UTC"), pytz.timezone("Asia/Istanbul"), pytz.timezone("US/Eastern"), pytz.timezone("Asia/Tokyo")] + +for _ in range(12): + random_ts = random.randint(mintime_ts, maxtime_ts) + randtom_tz = timezones[random.randint(0, len(timezones)-1)] + RANDOMTIME = datetime.datetime.fromtimestamp(random_ts, randtom_tz) + R_clear = datetime.datetime.fromtimestamp(RANDOMTIME.timestamp()) + + timestamp_text = str(R_clear) + timestamptz_text = str(RANDOMTIME) + timetz_text = str(RANDOMTIME.time()) + " " + RANDOMTIME.tzname() + time_text = str(RANDOMTIME.time()) + date_text = str(RANDOMTIME.date()) + dt = (R_clear-MINTIME) + interval_text = str(dt) + + timestamp_values += [timestamp_text] + timestamptz_values += [timestamptz_text] + time_values += [time_text] + timetz_values += [timetz_text] + date_values += [date_text] + interval_values += [interval_text] + +save_datafile("timestamp", timestamp_values) +save_datafile("timestamptz", timestamptz_values) +save_datafile("time", time_values) +save_datafile("timetz", timetz_values) +save_datafile("date", date_values) +save_datafile("interval", interval_values) diff --git a/contrib/try_convert/scripts/generate_test.py b/contrib/try_convert/scripts/generate_test.py new file mode 100644 index 00000000000..7b17be3d7c4 --- /dev/null +++ b/contrib/try_convert/scripts/generate_test.py @@ -0,0 +1,689 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import re + +from general import supported_types, string_types, typmod_types, typmod_lens + +def get_typemod_type(t, l): + if l is None: + return t + else: + return f'{t}({l})' + +def get_typemod_table(t, l): + if l is None: + return f'tt_{t}' + else: + return f'tt_{t}_{l}' + + +from general import uncomparable_types, has_corrupt_data + +from general import supported_extensions + + +print('Supported types:', ' '.join(supported_types)) + + +def remove_empty_lines(t): + return "\n".join([s for s in t.split("\n") if s]) + +### GET FUNCTION IDs + +from find_casts import get_pg_proc + +func_id_name = get_pg_proc() + + +### GET TYPE IDs + +from find_casts import get_pg_type + +type_name_id, type_id_name, _ = get_pg_type() + +supported_types_count = 0 + +print(f'Types found: {len(type_id_name)}, supported: {supported_types_count}') + + +### GET CONVERTS + +from find_casts import get_pg_cast + +casts = get_pg_cast(type_id_name, func_id_name) + +supported_cast_count = 0 + +print(f'Casts found: {len(casts)}, supported: {supported_cast_count}') + + +### HEADER & FOOTER + +test_header = \ + f'-- SCRIPT-GENERATED TEST for TRY_CONVERT\n' \ + f'-- Tests {supported_types_count} types of {len(type_id_name)} from pg_types.h\n' \ + f'-- Tests {supported_cast_count} cast of {len(casts)} from pg_cast.h\n' \ + f'create schema tryconvert;\n' \ + f'set search_path = tryconvert;\n' \ + f'-- start_ignore\n' \ + f'CREATE EXTENSION IF NOT EXISTS try_convert;\n' \ + f'-- end_ignore\n' + +for extension in supported_extensions: + test_header += \ + f'-- start_ignore\n' \ + f'CREATE EXTENSION IF NOT EXISTS {extension};\n' \ + f'-- end_ignore\n' + +test_header_out = test_header + +for type_name in supported_types: + add = \ + f'select add_type_for_try_convert(\'{type_name}\'::regtype);\n' + + out = \ + ' add_type_for_try_convert \n' \ + '--------------------------\n' \ + ' \n' \ + '(1 row)\n' \ + '\n' + + test_header += add + + test_header_out += add + out + +test_header_out = test_header_out[:-1] + +test_footer = 'reset search_path;' + + +### TRY_CONVERT_BY_SQL + +test_funcs = '' + +func_text = \ + f'CREATE FUNCTION try_convert_by_sql_text(_in text, INOUT _out ANYELEMENT, source_type text)\n' \ + f' LANGUAGE plpgsql AS\n' \ + f'$func$\n' \ + f' BEGIN\n' \ + f' EXECUTE format(\'SELECT %L::%s::%s\', $1, source_type, pg_typeof(_out))\n' \ + f' INTO _out;\n' \ + f' EXCEPTION WHEN others THEN\n' \ + f' -- do nothing: _out already carries default\n' \ + f' END\n' \ + f'$func$;\n' + +test_funcs += func_text + +func_text = \ + f'CREATE FUNCTION try_convert_by_sql_text_with_len_out(_in text, INOUT _out ANYELEMENT, source_type text, len_out int)\n' \ + f' LANGUAGE plpgsql AS\n' \ + f'$func$\n' \ + f' BEGIN\n' \ + f' EXECUTE format(\'SELECT %L::%s::%s(%s)\', $1, source_type, pg_typeof(_out), len_out::text)\n' \ + f' INTO _out;\n' \ + f' EXCEPTION WHEN others THEN\n' \ + f' -- do nothing: _out already carries default\n' \ + f' END\n' \ + f'$func$;\n' + +test_funcs += func_text + +for type_name in supported_types: + + func_text = \ + f'CREATE FUNCTION try_convert_by_sql_with_len_out(_in {type_name}, INOUT _out ANYELEMENT, len_out int)\n' \ + f' LANGUAGE plpgsql AS\n' \ + f'$func$\n' \ + f' BEGIN\n' \ + f' EXECUTE format(\'SELECT %L::{type_name}::%s(%s)\', $1, pg_typeof(_out), len_out::text)\n' \ + f' INTO _out;\n' \ + f' EXCEPTION WHEN others THEN\n' \ + f' -- do nothing: _out already carries default\n' \ + f' END\n' \ + f'$func$;\n' + + test_funcs += func_text + + func_text = \ + f'CREATE FUNCTION try_convert_by_sql(_in {type_name}, INOUT _out ANYELEMENT)\n' \ + f' LANGUAGE plpgsql AS\n' \ + f'$func$\n' \ + f' BEGIN\n' \ + f' EXECUTE format(\'SELECT %L::{type_name}::%s\', $1, pg_typeof(_out))\n' \ + f' INTO _out;\n' \ + f' EXCEPTION WHEN others THEN\n' \ + f' -- do nothing: _out already carries default\n' \ + f' END\n' \ + f'$func$;\n' + + test_funcs += func_text + +for source_type in supported_types: + for target_type in supported_types: + for source_typmod in typmod_lens: + if source_type not in typmod_types and source_typmod is not None: + continue + for target_typmod in typmod_lens: + if target_type not in typmod_types and target_typmod is not None: + continue + + source_name = source_type + if source_typmod is not None: + source_name += f'({source_typmod})' + + target_name = target_type + if target_typmod is not None: + target_name += f'({target_typmod})' + + + func_text = \ + f'CREATE FUNCTION try_convert_by_exception_{source_typmod}_{target_typmod}(_in {source_name}, d {target_name}) RETURNS {target_name}\n' \ + f' LANGUAGE plpgsql AS\n' \ + f'$func$\n' \ + f' BEGIN\n' \ + f' RETURN CAST(_in AS {target_name});\n' \ + f' EXCEPTION WHEN others THEN\n' \ + f' RETURN d;\n' \ + f' END\n' \ + f'$func$;\n' + + test_funcs += func_text + +### CREATE DATA + +test_load_data = '-- LOAD DATA\n' + +test_load_data += f'CREATE TABLE tt_temp (v text) DISTRIBUTED BY (v);\n' + +def copy_data(table_name, filename, type_name): + return f'DELETE FROM tt_temp;\n' \ + f'COPY tt_temp from \'@abs_srcdir@/{filename}\';\n' \ + f'INSERT INTO {table_name}(id, v) SELECT row_number() OVER(), v::{type_name} from tt_temp;' + +type_tables = {} + +def create_table(type_name, varlen=None): + table_name = get_typemod_table(type_name, varlen) + field_type = get_typemod_type(type_name, varlen) + + type_tables[type_name] = table_name + + load_data = f'CREATE TABLE {table_name} (id serial, v {field_type}) DISTRIBUTED BY (id);\n' + + filename = f'data/tt_{type_name}.data' + + load_data += copy_data(table_name, filename, field_type) + '\n' + + # load_data += f'SELECT * FROM {table_name};' + + return load_data + +def get_string_table(type_name, string_type, type_varlen=None, string_varlen=None): + + if type_varlen is not None and string_varlen is not None: + return f'tt_{string_type}_{string_varlen}_of_{type_name}_{type_varlen}' + elif type_varlen is not None: + return f'tt_{string_type}_of_{type_name}_{type_varlen}' + elif string_varlen is not None: + return f'tt_{string_type}_{string_varlen}_of_{type_name}' + + return f'tt_{string_type}_of_{type_name}' + +for type_name in supported_types: + + for type_varlen in typmod_lens: + if type_varlen is not None and type_name not in typmod_types: + continue + + test_load_data += create_table(type_name, type_varlen) + + for string_type in string_types: + for string_varlen in typmod_lens: + if string_varlen is not None and string_type not in typmod_types: + continue + + field_type = get_typemod_type(type_name, type_varlen) + string_field_type = get_typemod_type(string_type, string_varlen) + + table_name = get_string_table(type_name, string_type, type_varlen, string_varlen) + + load_data = f'CREATE TABLE {table_name} (id serial, v {string_field_type}) DISTRIBUTED BY (id);\n' + + cut = f'::{field_type}' if type_varlen is not None else '' + + load_data += f'INSERT INTO {table_name}(id, v) SELECT row_number() OVER(), v{cut}::{string_field_type} from tt_temp;\n' + + test_load_data += load_data + + if type_name in has_corrupt_data: + + for string_type in string_types: + for string_varlen in typmod_lens: + if string_varlen is not None and string_type not in typmod_types: + continue + + field_type = get_typemod_type(type_name, type_varlen) + string_field_type = get_typemod_type(string_type, string_varlen) + + corr_table_name = 'corr_' + get_string_table(type_name, string_type, type_varlen, string_varlen) + + load_data = f'CREATE TABLE {corr_table_name} (id serial, v {string_field_type}) DISTRIBUTED BY (id);\n' + + filename = f'data/corr_{type_name}.data' + + load_data += copy_data(corr_table_name, filename, string_field_type) + '\n' + + test_load_data += load_data + + + + +## GET DATA + +def get_data(type_name): + return type_tables[type_name] + +def get_len_from_data(type_name): + f = open(f'data/tt_{type_name}.data') + return(len(f.read().split('\n'))) + +def get_len_from_corr_data(type_name): + f = open(f'data/corr_{type_name}.data') + return(len(f.read().split('\n'))) + +def get_from_data(type_name, i = None): + f = open(f'data/tt_{type_name}.data') + values = f.read().split('\n') + if i is None: + return content + return values[i] + +## TEST + +def create_test(source_name, target_name, test_data, default='NULL', source_varlen=None, target_varlen=None, source_count=0): + + test_filter = 'v1 is distinct from v2' if target_name not in uncomparable_types else 'v1::text is distinct from v2::text' + test_filter_not = 'v1 is not distinct from v2' if target_name not in uncomparable_types else 'v1::text is not distinct from v2::text' + + target_name_1 = get_typemod_type(target_name, target_varlen) + + try_convert_sql = f'try_convert_by_exception_{source_varlen}_{target_varlen}(v, {default}::{target_name_1})' + + query = \ + f'select * from (' \ + f'select ' \ + f'try_convert(v, {default}::{target_name_1}) as v1, ' \ + f'{try_convert_sql} as v2' \ + f', v' \ + f' from {test_data}' \ + f') as t(v1, v2, v) where {test_filter};' + result = \ + ' v1 | v2 | v \n' \ + '----+----+---\n' \ + '(0 rows)\n' + + query_not = \ + f'select count(*) from (' \ + f'select ' \ + f'try_convert(v, {default}::{target_name_1}) as v1, ' \ + f'{try_convert_sql} as v2' \ + f' from {test_data}' \ + f') as t(v1, v2) where {test_filter_not};' + result_not = \ + ' count \n' \ + '-------\n' \ + f' {source_count}\n' \ + '(1 row)\n' + + input_source = query + '\n' + query_not + output_source = remove_empty_lines(query) + '\n' + result + '\n' + remove_empty_lines(query_not) + '\n' + result_not + + return input_source, output_source + + +### CAST to & from text + +text_tests_in = [] +text_tests_out = [] + +default_value = 'NULL' + +for string_type in string_types: + for string_varlen in typmod_lens: + if string_varlen is not None and type_name not in typmod_types: + continue + + for type_name in supported_types: + for type_varlen in typmod_lens: + if type_varlen is not None and type_name not in typmod_types: + continue + + test_type_table = get_typemod_table(type_name, type_varlen) + + text_type_table = get_string_table(type_name, string_type, type_varlen, string_varlen) + + test_corrupted_text_data = f'(select (\'!@#%^&*\' || v || \'!@#%^&*\') from {text_type_table}) as t(v)' + + data_count = get_len_from_data(type_name) + + to_text_in, to_text_out = create_test( + type_name, string_type, + test_type_table, default_value, + type_varlen, string_varlen, + data_count + ) + from_text_in, from_text_out = create_test( + string_type, type_name, + text_type_table, default_value, + string_varlen, type_varlen, + data_count + ) + from_corrupted_text_in, from_corrupted_text_out = create_test( + string_type, type_name, + test_corrupted_text_data, default_value, + string_varlen, type_varlen, + data_count + ) + + text_tests_in += [to_text_in, from_text_in] + text_tests_out += [to_text_out, from_text_out] + + text_tests_in += [from_corrupted_text_in] + text_tests_out += [from_corrupted_text_out] + + data_count = get_len_from_data(type_name) + + if type_name in has_corrupt_data: + + corr_text_type_table = 'corr_' + text_type_table + + data_count = get_len_from_corr_data(type_name) + + from_corr_in, from_corr_out = create_test( + string_type, type_name, + corr_text_type_table, default_value, + string_varlen, type_varlen, + data_count + ) + + text_tests_in += [from_corr_in] + text_tests_out += [from_corr_out] + + +# print(text_tests_in[0]) +# print(text_tests_in[1]) + + +### CAST from pg_cast + +function_tests_in = [] +function_tests_out = [] + +type_casts = [(source_name, target_name) for (source_name, target_name, method) in casts] + +for source_name, target_name in type_casts: + if (source_name not in supported_types or target_name not in supported_types): + continue + + dd = get_from_data(target_name, 0).translate(str.maketrans('', '', '\'')) + d = f'\'{dd}\'' + + for default in ['NULL', d]: + + for source_varlen in typmod_lens: + if source_varlen is not None and source_name not in typmod_types: + continue + + test_table = get_typemod_table(source_name, source_varlen) + + for target_varlen in typmod_lens: + if target_varlen is not None and target_name not in typmod_types: + continue + + data_count = get_len_from_data(source_name) + + test_in, test_out = create_test( + source_name, target_name, + test_table, default, + source_varlen, target_varlen, + data_count + ) + + function_tests_in += [test_in] + function_tests_out += [test_out] + + +# print(function_tests_in[0]) + + +### DEFAULTS TEST + +# for type_name in supported_types: + +# query = f'SELECT try_convert({}::{}, {get_from_data(type_name, 0)}::{type_name});' + +### ONE MILLION ERRORS + +test_million = '' + +test_million_data = \ + 'DROP TABLE IF EXISTS text_ints; CREATE TABLE text_ints (v text) DISTRIBUTED BY (v);\n' \ + 'INSERT INTO text_ints(v) SELECT (random()*1000)::int4::text FROM generate_series(1,1000000);\n' \ + 'DROP TABLE IF EXISTS text_error_ints; CREATE TABLE text_error_ints (v text) DISTRIBUTED BY (v);\n' \ + 'INSERT INTO text_error_ints(v) SELECT (random()*1000000 + 1000000)::int8::text FROM generate_series(1,1000000);\n' \ + 'DROP TABLE IF EXISTS int4_ints; CREATE TABLE int4_ints (v int4) DISTRIBUTED BY (v);\n' \ + 'INSERT INTO int4_ints(v) SELECT (random()*1000)::int4 FROM generate_series(1,1000000);\n' \ + 'DROP TABLE IF EXISTS int4_error_ints; CREATE TABLE int4_error_ints (v int4) DISTRIBUTED BY (v);\n' \ + 'INSERT INTO int4_error_ints(v) SELECT (random()*1000000 + 1000000)::int4 FROM generate_series(1,1000000);\n' + +test_million_query1 = \ + 'SELECT count(*) FROM (SELECT try_convert(v, NULL::int2) as v FROM text_ints) as t(v) WHERE v IS NOT NULL;\n' +test_million_query2 = \ + 'SELECT count(*) FROM (SELECT try_convert(v, NULL::int2) as v FROM text_error_ints) as t(v) WHERE v IS NULL;\n' + +test_million_query3 = \ + 'SELECT count(*) FROM (SELECT try_convert(v, NULL::int2) as v FROM int4_ints) as t(v) WHERE v IS NOT NULL;\n' +test_million_query4 = \ + 'SELECT count(*) FROM (SELECT try_convert(v, NULL::int2) as v FROM int4_error_ints) as t(v) WHERE v IS NULL;\n' + +test_million_result = \ + ' count \n' \ + '---------\n' \ + ' 1000000\n' \ + '(1 row)\n' \ + +test_million_in = test_million_data + test_million_query1 + test_million_query2 + test_million_query3 + test_million_query4 +test_million_out = test_million_data + \ + test_million_query1 + test_million_result + '\n' + \ + test_million_query2 + test_million_result + '\n' + \ + test_million_query3 + test_million_result + '\n' + \ + test_million_query4 + test_million_result + +### NESTED CASTS + +value = '42::int4' + +for level in range(100): + value = f'try_convert(try_convert({value}, NULL::text), NULL::int4)' + + +test_nested_query = f'select {value} as v;\n' + +test_nested_result = \ + ' v \n' \ + '----\n' \ + ' 42\n' \ + '(1 row)\n' \ + +test_nested_in = test_nested_query +test_nested_out = test_nested_query + test_nested_result + +edge_case_queries = [ + # --- NULL FALLBACK + NUMERIC --- + "select try_convert('42d'::text, NULL::numeric(38,2)) is null as r;", + "select try_convert('42d'::text, 0::numeric(38,2)) = 0 as r;", + "select try_convert('42.123'::text, NULL::numeric(38,2)) = 42.12 as r;", + + # --- NULL source --- + "select try_convert(NULL::text, 42::int) is null as r;", + "select try_convert(NULL::text, NULL::int) is null as r;", + "select try_convert(NULL::int, 7::int) is null as r;", + + # --- Fallback value must respect target typmod (regression for is_failed reset) --- + "select try_convert('bad'::text, 3.14159::numeric(10,2)) = 3.14 as r;", + "select try_convert('bad'::text, 1::numeric(4,2)) = 1.00 as r;", + + # --- RELABEL path (same base type) --- + "select try_convert('hello'::text, NULL::text) = 'hello' as r;", + "select try_convert('abcdefgh'::varchar(20), NULL::varchar(5)) = 'abcde' as r;", + "select try_convert(42.567::numeric(10,3), NULL::numeric(5,1)) = 42.6 as r;", + + # --- Numeric overflow / rounding / typmod --- + "select try_convert('99999.999'::text, NULL::numeric(5,2)) is null as r;", + "select try_convert('-99999.999'::text, NULL::numeric(5,2)) is null as r;", + "select try_convert('0.001'::text, NULL::numeric(5,2)) = 0.00 as r;", + "select try_convert('0.005'::text, NULL::numeric(5,2)) = 0.01 as r;", + + # --- Empty / whitespace strings --- + "select try_convert(''::text, NULL::int) is null as r;", + "select try_convert(''::text, 0::int) = 0 as r;", + "select try_convert(' '::text, 0::int) = 0 as r;", + "select try_convert(' 42 '::text, NULL::int) = 42 as r;", + + # --- Signed numbers --- + "select try_convert('+42'::text, NULL::int) = 42 as r;", + "select try_convert('-42'::text, NULL::int) = -42 as r;", + + # --- Integer overflow / wide types --- + "select try_convert('99999999999'::text, NULL::int) is null as r;", + "select try_convert('99999999999'::text, NULL::bigint) = 99999999999 as r;", + "select try_convert('1e10'::text, NULL::bigint) is null as r;", + "select try_convert('1e100'::text, NULL::int) is null as r;", + + # --- Float special values --- + "select try_convert('NaN'::text, NULL::float8) = 'NaN'::float8 as r;", + "select try_convert('Infinity'::text, NULL::float8) = 'Infinity'::float8 as r;", + "select try_convert('-Infinity'::text, NULL::float8) = '-Infinity'::float8 as r;", + + # --- Date / time invalid inputs --- + "select try_convert('2026-13-01'::text, NULL::date) is null as r;", + "select try_convert('2026-02-30'::text, NULL::date) is null as r;", + "select try_convert('not-a-date'::text, NULL::date) is null as r;", + "select try_convert('2026-04-20'::text, NULL::date) = '2026-04-20'::date as r;", + + # --- Boolean parsing --- + "select try_convert('true'::text, NULL::bool) = true as r;", + "select try_convert('t'::text, NULL::bool) = true as r;", + "select try_convert('1'::text, NULL::bool) = true as r;", + "select try_convert('yes'::text, NULL::bool) = true as r;", + "select try_convert('maybe'::text, NULL::bool) is null as r;", + "select try_convert('maybe'::text, false::bool) = false as r;", + + # --- Nested calls --- + # '42.9' -> numeric succeeds, but int parser doesn't accept decimals -> NULL + "select try_convert(try_convert('42.9'::text, NULL::numeric)::text, NULL::int) is null as r;", + # Inner returns NULL; outer receives NULL source, returns NULL (not the fallback) + # This documents current behaviour: NULL source is short-circuited before fallback. + "select try_convert(try_convert('bad'::text, NULL::int)::text, -1::int) is null as r;", + + # --- JSON (if supported) --- + "select try_convert('{\"a\":1}'::text, NULL::json) is not null as r;", + "select try_convert('{bad json}'::text, NULL::json) is null as r;", +] + +test_edge_cases_result = \ + ' r \n' \ + '---\n' \ + ' t\n' \ + '(1 row)\n' + +test_edge_cases_in = '\n'.join(edge_case_queries) + '\n' +test_edge_cases_out = '\n'.join( + q + '\n' + test_edge_cases_result for q in edge_case_queries +) + + +### EDGE CASE: try_convert inside PL/pgSQL (SPI context regression) + +test_spi_in = \ + "DO $$\n" \ + "DECLARE\n" \ + " r int;\n" \ + "BEGIN\n" \ + " SELECT try_convert('42'::text, 0::int) INTO r;\n" \ + " IF r <> 42 THEN RAISE EXCEPTION 'expected 42, got %', r; END IF;\n" \ + " SELECT try_convert('bad'::text, -1::int) INTO r;\n" \ + " IF r <> -1 THEN RAISE EXCEPTION 'expected -1, got %', r; END IF;\n" \ + "END$$;" + +test_spi_out = test_spi_in + + +### CONSTRUCT TEST + +test_str = '\n'.join([ + test_header, \ + # FUNCTIONS + test_funcs, \ + # CREATE DATA + test_load_data, \ + '-- TEXT TESTS', \ + '\n'.join(text_tests_in), \ + '-- FUNCTION TESTS', \ + '\n'.join(function_tests_in), \ + '-- MILLION TESTS', \ + test_million_in, + '-- NESTED TESTS', \ + test_nested_in, + '-- EDGE CASES', \ + test_edge_cases_in, + '-- SPI CONTEXT', \ + test_spi_in, + test_footer + ]) + '\n' + +test_f = open('input/try_convert.source', 'w') +test_f.write(test_str) + + +test_str = '\n'.join([ + test_header_out, \ + # FUNCTIONS + remove_empty_lines(test_funcs), \ + # CREATE DATA + remove_empty_lines(test_load_data), \ + '-- TEXT TESTS', \ + '\n'.join(text_tests_out), \ + '-- FUNCTION TESTS', \ + '\n'.join(function_tests_out), \ + '-- MILLION TESTS', \ + test_million_out, + '-- NESTED TESTS', \ + test_nested_out, + '-- EDGE CASES', \ + test_edge_cases_out, + '-- SPI CONTEXT', \ + test_spi_out, + remove_empty_lines(test_footer) + ]) + '\n' + +test_f = open('output/try_convert.source', 'w') +test_f.write(test_str) diff --git a/contrib/try_convert/scripts/verify.py b/contrib/try_convert/scripts/verify.py new file mode 100644 index 00000000000..8eb8d0b65ae --- /dev/null +++ b/contrib/try_convert/scripts/verify.py @@ -0,0 +1,368 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +##################################################### +# # +# Verifies no ereport called while convertation # +# # +##################################################### + +# Need also to verify context forwarding + +import re +import os, glob + +DEBUG_FLAG = True +PRINT_ALL_FLAG = False + +from general import source_filenames +from general import supported_types + +def filter_supported_casts(casts): + supported_casts = [] + for cast in casts: + if cast[0] in supported_types and cast[1] in supported_types: + supported_casts += [cast] + return supported_casts + +# Get casts from pg_cast + +from find_casts import get_pg_cast, get_pg_proc, get_pg_type + +func_id_name = get_pg_proc() +type_name_id, type_id_name, type_io_funcs = get_pg_type() + +pg_casts = filter_supported_casts(get_pg_cast(type_id_name, func_id_name)) + +# Get casts from extensions + +from find_casts import get_extensions + + +extension_casts, extension_sql_functions = get_extensions() + +casts = pg_casts + extension_casts + + +if DEBUG_FLAG: + print(f'FOUND {len(casts)} CASTs') + if PRINT_ALL_FLAG: + for l in sorted(casts): + print(l) + + +# +# Get functions used to cast +# + +# Get INOUT functions + +required_in_funcs = {} +required_out_funcs = {} + +for cast in casts: + if cast[2] == 'WITH INOUT': + required_out_funcs[cast[0]] = True + required_in_funcs[cast[1]] = True + +if DEBUG_FLAG: + print(f'REQUIRED {len(required_in_funcs)} _IN FUNCTIONS') + if PRINT_ALL_FLAG: + for f in sorted(required_in_funcs): + print(f) + +if DEBUG_FLAG: + print(f'REQUIRED {len(required_out_funcs)} _OUT FUNCTIONS') + if PRINT_ALL_FLAG: + for f in sorted(required_out_funcs): + print(f) + + +# Get functions from CREATE CAST + +required_funcs = {} + +for cast in pg_casts: + if cast[2] != 'WITH INOUT' and cast[2] != 'WITHOUT FUNCTION': + required_funcs[cast[2]] = cast + + +from find_casts import find_create_function_in_text + +sql_funcs = extension_sql_functions + +# print(sql_funcs) + +for cast in extension_casts: + if cast[2] != 'WITH INOUT' and cast[2] != 'WITHOUT FUNCTION': + sql_func_name = cast[2] + + m = re.match('(\w+)\(', sql_func_name) + if m is not None: + sql_func_name = m[1] + + c_func = "Not found" + + if sql_func_name in sql_funcs: + c_func = sql_funcs[sql_func_name] + + # print(sql_func_name, c_func) + + required_funcs[c_func] = sql_func_name + +if DEBUG_FLAG: + print(f'REQUIRED {len(required_funcs)} FUNCTIONS FROM CREATE CAST') + if PRINT_ALL_FLAG: + for f in sorted(required_funcs): + print(f) + +required_funcs_list = list(required_funcs) + list(required_in_funcs) + list(required_out_funcs) + +for l in type_io_funcs: + if l in supported_types: + required_funcs_list += type_io_funcs[l] + +# +# Load functions bodies +# + +# load convert function + +from find_calls import find_functions + +convert_functions = [] + +for root, subdirs, files in os.walk('../..'): + for filename in files: + file_path = os.path.join(root, filename) + if filename[-2:] == '.c': + # if filename in source_filenames: + # print(file_path) + with open(file_path, 'r') as f: + content = f.read() + + funcs = find_functions(required_funcs_list, content) + + convert_functions += funcs + + # print(file_path, len(funcs)) + +loaded_convert_functions = {} + +from find_calls import remove_comments + +for name, return_type, args, body in convert_functions: + if return_type == 'Datum' and args == 'PG_FUNCTION_ARGS': + body = remove_comments(body) + loaded_convert_functions[name] = body + +for rf in required_funcs: + if rf not in loaded_convert_functions: + print(f'body for {rf}({required_funcs[rf]}) not found') + + +if DEBUG_FLAG: + print(f'FOUND {len(loaded_convert_functions)} FUNCTIONS BODIES') + +# load safe functions + +from find_calls import find_safe_functions + +safe_functions = [] + +null_functions = [] + +for root, subdirs, files in os.walk('../..'): + for filename in files: + file_path = os.path.join(root, filename) + if filename[-2:] == '.c': + # if filename in source_filenames: + # print(file_path) + with open(file_path, 'r') as f: + content = f.read() + + funcs = find_safe_functions(content) + + safe_functions += funcs + + +loaded_safe_functions = {} +loaded_null_functions = {} + +for name, return_type, args, body in safe_functions: + loaded_safe_functions[name] = remove_comments(body) + + if re.search(r'bool', return_type) is None: + print(f' WARNING: safe function {name} returns result not bool') + + m = re.match('(\w+)Safe', name) + if m is not None: + loaded_null_functions[m[1]] = 1 + + m = re.match('(\w+)_safe', name) + if m is not None: + loaded_null_functions[m[1]] = 1 + +if DEBUG_FLAG: + print(f'FOUND {len(loaded_safe_functions)} SAFE FUNCTIONS BODIES') + +if DEBUG_FLAG: + print(f'REQUIRED {len(loaded_null_functions)} SAFE FUNCTIONS VARIANTS') + +loaded_functions = dict(list(loaded_convert_functions.items()) + list(loaded_safe_functions.items())) + +# +# Check functions don't call unsafe ereport +# + +from find_calls import get_all_functions_with + +ereport_functions = get_all_functions_with('ereport\(ERROR,') + +unsafe_convert_functions = {} + +for func_name in loaded_functions: + body = loaded_functions[func_name] + + pattern_call = '(\w+)\s*\(([\s\S]*?)\)' + + for token_match in re.finditer(pattern_call, body): + + token = token_match[1] + + args = token_match[2] + + # if (token == 'ereport'): + # print(token, args[:5]) + + if token in ereport_functions or ((token == 'ereport' or token == 'elog') and args[:5] == 'ERROR'): + print(f' WARNING: call unsafe function {token} in {func_name}') + unsafe_convert_functions[func_name] = token + continue + +if DEBUG_FLAG: + print(f'FOUND {len(unsafe_convert_functions)} UNSAFE CONVERT FUNCTIONS') + + +# +# Check context forwarding don't call unsafe variants +# + +unsafe_variants = set(loaded_null_functions.keys()) + +unsafe_variant_usage = {} +unsafe_variant_usage_count = 0 +unwrapped_safe_usage = {} +unwrapped_safe_usage_count = 0 +wrong_wrap_usage = {} +wrong_wrap_usage_count = 0 + +for func_name in loaded_functions: + body = loaded_functions[func_name] + + pattern_call = '\w+' + + for token_match in re.finditer(pattern_call, body): + + token = token_match[0] + + l = max(0, token_match.start() - 40) + r = min(token_match.end() + 150, len(body)) + + line = body[l:r] + + if token in unsafe_variants: + print(f' WARNING: call unsafe variant of function {token} in {func_name}') + unsafe_variant_usage[func_name] = token + unsafe_variant_usage_count += 1 + continue + + if token == 'ereturn' and re.search(r'ereturn\(fcinfo', line): + print(f' WARNING: ereturn cannot be run at fcinfo context in {func_name}, use PG_ERETURN') + + if token == 'return' and func_name in loaded_safe_functions: + m = re.match(r'return\s*([\s\S]*?);', body[token_match.start():r]) + if m[1] != 'true': + print(f' WARNING: in safe function {func_name}: "return" used only with "true", not "{m[1]}"') + + + if token in loaded_functions: + + # if_wrapper_pattern = rf'if \(!{token}\([\s\S]+?, (?:escontext|fcinfo->context)\)\)' + void_wrapper_pattern = rf'(void) {token}\([\s\S]+?, NULL\);' + safe_call_wrapper_pattern = rf'safe_call\({token}, \([\s\S]+?, (?:escontext|fcinfo->context)\)\);' + safe_call_with_free_wrapper_pattern = rf'safe_call_with_free\({token}, \([\s\S]+?, (?:escontext|fcinfo->context)\), \{{[\s\S]+?\}}\);' + pg_safe_call_wrapper_pattern = rf'PG_SAFE_CALL\({token}, \([\s\S]+?, (?:escontext|fcinfo->context)\)\);' + return_wrapper_pattern = rf'return {token}\([\s\S]+?, (?:escontext|fcinfo->context)\);' + direct_call_wrapper_pattern = rf'DirectFunctionCall1Safe\({token}, [\s\S]+?, (?:escontext|fcinfo->context)\);' + + def unite_patterns(l): + return '|'.join(l) + + search_pattern = unite_patterns([ + # if_wrapper_pattern, + safe_call_wrapper_pattern, + safe_call_with_free_wrapper_pattern, + pg_safe_call_wrapper_pattern, + return_wrapper_pattern, + direct_call_wrapper_pattern + ]) + + if not re.search(search_pattern, line): + # print(func_name, token_match, line) + print(f' WARNING: call unwrapped safe function {token} in {func_name}') + unwrapped_safe_usage[func_name] = token + unwrapped_safe_usage_count += 1 + continue + + pg_sc_patterns = unite_patterns([ + pg_safe_call_wrapper_pattern, + direct_call_wrapper_pattern, + ]) + if func_name in loaded_convert_functions and not re.search(pg_sc_patterns, line): + print(f' WARNING: wrong wrapped safe function {token} in PG_FUNCTION {func_name}') + wrong_wrap_usage[func_name] = token + wrong_wrap_usage_count += 1 + continue + + sc_patterns = unite_patterns([ + safe_call_wrapper_pattern, + safe_call_with_free_wrapper_pattern, + return_wrapper_pattern, + ]) + if func_name in loaded_safe_functions and not re.search(sc_patterns, line): + print(f' WARNING: wrong wrapped safe function {token} in SAFE {func_name}') + wrong_wrap_usage[func_name] = token + wrong_wrap_usage_count += 1 + continue + + + +if DEBUG_FLAG: + print(f'FOUND {unsafe_variant_usage_count} UNSAFE VARIANT USAGES') + +if DEBUG_FLAG: + print(f'FOUND {unwrapped_safe_usage_count} UNWRAPPED SAFE FUNCTION USAGES') + +if DEBUG_FLAG: + print(f'FOUND {wrong_wrap_usage_count} WRONG WRAP USAGES') + + +# print(loaded_functions['date_in']) +# print(loaded_functions['timestamptz_interval_bound']) diff --git a/contrib/try_convert/try_convert--1.0.sql b/contrib/try_convert/try_convert--1.0.sql new file mode 100644 index 00000000000..785a66a6f6b --- /dev/null +++ b/contrib/try_convert/try_convert--1.0.sql @@ -0,0 +1,91 @@ +/* contrib/try_convert/try_convert--1.0.sql */ + +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION try_convert" to load this file. \quit + +/* *********************************************** + * try_convert function for PostgreSQL + * *********************************************** */ + +/* generic file access functions */ + +CREATE FUNCTION try_convert(text, anyelement) +RETURNS anyelement +AS 'MODULE_PATHNAME', 'try_convert' +LANGUAGE C; + + +CREATE OR REPLACE FUNCTION add_type_for_try_convert(type regtype) + RETURNS void + LANGUAGE plpgsql AS +$func$ +BEGIN + EXECUTE 'CREATE OR REPLACE FUNCTION try_convert(' || type || ', anyelement) + RETURNS anyelement + AS ''MODULE_PATHNAME'', ''try_convert'' + LANGUAGE C;'; +END +$func$; + +-- NUMBERS +select add_type_for_try_convert('int2'::regtype); +select add_type_for_try_convert('int4'::regtype); +select add_type_for_try_convert('int8'::regtype); +select add_type_for_try_convert('float4'::regtype); +select add_type_for_try_convert('float8'::regtype); +select add_type_for_try_convert('numeric'::regtype); +select add_type_for_try_convert('complex'::regtype); + +-- TIME +select add_type_for_try_convert('date'::regtype); +select add_type_for_try_convert('time'::regtype); +select add_type_for_try_convert('timetz'::regtype); +select add_type_for_try_convert('timestamp'::regtype); +select add_type_for_try_convert('timestamptz'::regtype); +select add_type_for_try_convert('interval'::regtype); + +-- CHARACTER +select add_type_for_try_convert('char'::regtype); +select add_type_for_try_convert('bpchar'::regtype); +select add_type_for_try_convert('varchar'::regtype); +select add_type_for_try_convert('text'::regtype); + +-- BIT STRING +select add_type_for_try_convert('bit'::regtype); +select add_type_for_try_convert('varbit'::regtype); + +select add_type_for_try_convert('bool'::regtype); + +select add_type_for_try_convert('money'::regtype); + +select add_type_for_try_convert('uuid'::regtype); + +-- GEOMETRY +select add_type_for_try_convert('point'::regtype); + +-- IP/MAC +select add_type_for_try_convert('cidr'::regtype); +select add_type_for_try_convert('inet'::regtype); +select add_type_for_try_convert('macaddr'::regtype); + +-- OBJ +select add_type_for_try_convert('json'::regtype); +select add_type_for_try_convert('jsonb'::regtype); +select add_type_for_try_convert('xml'::regtype); diff --git a/contrib/try_convert/try_convert.c b/contrib/try_convert/try_convert.c new file mode 100644 index 00000000000..8c9afccefc3 --- /dev/null +++ b/contrib/try_convert/try_convert.c @@ -0,0 +1,564 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * try_convert.c + * Error-safe type cast function + * + * try_convert(source_value, default_value) casts source_value to the type of + * default_value. Whenever the cast fails because of the data being converted, + * default_value is returned instead of an error being raised. + * + * The conversion to use is looked up the same way the parser does it in + * coerce_type(), see src/backend/parser/parse_coerce.c. + * + * IDENTIFICATION + * contrib/try_convert/try_convert.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "catalog/pg_cast.h" +#include "catalog/pg_type.h" +#include "funcapi.h" +#include "nodes/nodeFuncs.h" +#include "parser/parse_coerce.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/syscache.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(try_convert); + +/* + * How a value has to be converted; a subset of the CoercionPathType values + * used by the parser. + */ +typedef enum ConversionType +{ + CONVERSION_TYPE_FUNC, + CONVERSION_TYPE_RELABEL, + CONVERSION_TYPE_VIA_IO, + CONVERSION_TYPE_ARRAY, + CONVERSION_TYPE_NONE +} ConversionType; + +static ConversionType find_conversion_way(Oid targetTypeId, Oid sourceTypeId, + Oid *funcId); +static ConversionType find_typmod_conversion_function(Oid typeId, Oid *funcId); +static void report_conversion_error(MemoryContext oldcontext, bool *is_failed); +static Datum convert_from_function(Datum value, int32 typmod, Oid funcId, + bool *is_failed); +static Datum convert_via_io(Datum value, Oid sourceTypeId, Oid targetTypeId, + bool *is_failed); +static int32 get_call_expr_argtypmod(Node *expr, int argnum); +static int32 get_fn_expr_argtypmod(FmgrInfo *flinfo, int argnum); +static Datum convert(Datum value, ConversionType conversion_type, Oid funcId, + Oid sourceTypeId, Oid targetTypeId, int32 targetTypMod, + bool *is_failed); +static Datum convert_type_typmod(Datum value, int32 sourceTypMod, + Oid targetTypeId, int32 targetTypMod, + bool *is_failed); + + +/* + * Determine how to convert sourceTypeId to targetTypeId, mirroring + * find_coercion_pathway() without the coercion context: try_convert() always + * performs an explicit cast. + * + * Returns CONVERSION_TYPE_NONE if no conversion is possible. *funcId is set + * to the conversion function when one is needed, to InvalidOid otherwise. + */ +static ConversionType +find_conversion_way(Oid targetTypeId, Oid sourceTypeId, Oid *funcId) +{ + ConversionType result = CONVERSION_TYPE_NONE; + HeapTuple tuple; + + *funcId = InvalidOid; + + /* + * Both types have to be known. An invalid type OID means the caller could + * not resolve the argument type, and the lookups below -- TypeCategory() + * in particular -- do not accept one. + */ + if (!OidIsValid(sourceTypeId) || !OidIsValid(targetTypeId)) + return CONVERSION_TYPE_NONE; + + /* Perhaps the types are domains; if so, look at their base types */ + sourceTypeId = getBaseType(sourceTypeId); + targetTypeId = getBaseType(targetTypeId); + + /* Domains are always coercible to and from their base type */ + if (sourceTypeId == targetTypeId) + return CONVERSION_TYPE_RELABEL; + + /* Look in pg_cast */ + tuple = SearchSysCache2(CASTSOURCETARGET, + ObjectIdGetDatum(sourceTypeId), + ObjectIdGetDatum(targetTypeId)); + + if (HeapTupleIsValid(tuple)) + { + Form_pg_cast castForm = (Form_pg_cast) GETSTRUCT(tuple); + + switch (castForm->castmethod) + { + case COERCION_METHOD_FUNCTION: + *funcId = castForm->castfunc; + result = CONVERSION_TYPE_FUNC; + break; + case COERCION_METHOD_INOUT: + result = CONVERSION_TYPE_VIA_IO; + break; + case COERCION_METHOD_BINARY: + result = CONVERSION_TYPE_RELABEL; + break; + default: + elog(ERROR, "unrecognized castmethod: %d", + (int) castForm->castmethod); + break; + } + + ReleaseSysCache(tuple); + } + else + { + /* + * If there's no pg_cast entry, perhaps we are dealing with a pair of + * array types. If so, and if the element types have a suitable cast, + * report that we can coerce with an ArrayCoerceExpr. + * + * Note that the source type can be a domain over array, but not the + * target, because ArrayCoerceExpr won't check domain constraints. + * + * Hack: disallow coercions to oidvector and int2vector, which + * otherwise tend to capture coercions that should go to "real" array + * types. We want those types to be considered "real" arrays for many + * purposes, but not this one. (Also, ArrayCoerceExpr isn't + * guaranteed to produce an output that meets the restrictions of + * these datatypes, such as being 1-dimensional.) + */ + if (targetTypeId != OIDVECTOROID && targetTypeId != INT2VECTOROID) + { + Oid targetElem; + Oid sourceElem; + + if ((targetElem = get_element_type(targetTypeId)) != InvalidOid && + (sourceElem = get_base_element_type(sourceTypeId)) != InvalidOid) + { + ConversionType elempathtype; + Oid elemfuncid; + + elempathtype = find_conversion_way(targetElem, + sourceElem, + &elemfuncid); + if (elempathtype != CONVERSION_TYPE_NONE && + elempathtype != CONVERSION_TYPE_ARRAY) + { + *funcId = elemfuncid; + if (elempathtype == CONVERSION_TYPE_VIA_IO) + result = CONVERSION_TYPE_VIA_IO; + else + result = CONVERSION_TYPE_ARRAY; + } + } + } + + /* + * If we still haven't found a possibility, consider automatic casting + * using I/O functions. We allow assignment casts to string types and + * explicit casts from string types to be handled this way. (The + * CoerceViaIO mechanism is a lot more general than that, but this is + * all we want to allow in the absence of a pg_cast entry.) It would + * probably be better to insist on explicit casts in both directions, + * but this is a compromise to preserve something of the pre-8.3 + * behavior that many types had implicit (yipes!) casts to text. + */ + if (result == CONVERSION_TYPE_NONE) + { + if (TypeCategory(targetTypeId) == TYPCATEGORY_STRING) + result = CONVERSION_TYPE_VIA_IO; + else if (TypeCategory(sourceTypeId) == TYPCATEGORY_STRING) + result = CONVERSION_TYPE_VIA_IO; + } + } + + return result; +} + +/* + * Look up the length coercion function of a type, that is the cast from the + * type to itself, mirroring find_typmod_coercion_function(). + * + * Returns CONVERSION_TYPE_NONE when the type has no length coercion function, + * in which case the value has to be left alone. + */ +static ConversionType +find_typmod_conversion_function(Oid typeId, Oid *funcId) +{ + ConversionType result = CONVERSION_TYPE_NONE; + HeapTuple tuple; + + *funcId = InvalidOid; + + /* Look in pg_cast */ + tuple = SearchSysCache2(CASTSOURCETARGET, + ObjectIdGetDatum(typeId), + ObjectIdGetDatum(typeId)); + + if (HeapTupleIsValid(tuple)) + { + Form_pg_cast castForm = (Form_pg_cast) GETSTRUCT(tuple); + + *funcId = castForm->castfunc; + ReleaseSysCache(tuple); + + /* + * A binary-coercible self-cast carries no function, so there is + * nothing we could call to apply the typmod. + */ + if (OidIsValid(*funcId)) + result = CONVERSION_TYPE_FUNC; + } + + return result; +} + +/* + * Common tail of the PG_CATCH() blocks below. + * + * The conversion functions called by this module report a failure the only way + * they can, by throwing an error, so the only way to keep going is to catch it. + * Ideally we would use the "soft" error handling infrastructure added by + * PostgreSQL 17 (commit ccff2d20ed) instead, which lets an input function + * report a conversion failure without throwing; that requires converting the + * datatype input functions first, so until then we trap the error here. + * + * Trapping an error outside of a subtransaction is only safe as long as the + * called function leaves no global state behind, which holds for the cast and + * type input/output functions we call. Errors that do not come from the data + * being converted must not be swallowed, so query cancellation and assertion + * failures are re-thrown, following what plpgsql does for "EXCEPTION WHEN + * others". + */ +static void +report_conversion_error(MemoryContext oldcontext, bool *is_failed) +{ + int sqlerrcode = geterrcode(); + + if (sqlerrcode == ERRCODE_QUERY_CANCELED || + sqlerrcode == ERRCODE_ASSERT_FAILURE) + PG_RE_THROW(); + + MemoryContextSwitchTo(oldcontext); + FlushErrorState(); + + *is_failed = true; +} + +/* + * Convert a value by calling the cast function funcId. + */ +static Datum +convert_from_function(Datum value, int32 typmod, Oid funcId, bool *is_failed) +{ + MemoryContext oldcontext = CurrentMemoryContext; + volatile Datum res = (Datum) 0; + + PG_TRY(); + { + /* + * Cast functions take either one argument or three, the extra ones + * being the target typmod and the explicit-cast flag. Passing three + * arguments to a one-argument function is harmless, the callee simply + * ignores them. + */ + res = OidFunctionCall3(funcId, + value, + Int32GetDatum(typmod), + BoolGetDatum(true)); + } + PG_CATCH(); + { + report_conversion_error(oldcontext, is_failed); + } + PG_END_TRY(); + + return res; +} + +/* + * Convert a value by running it through the output function of the source type + * and the input function of the target type. + * + * The typmod is not applied here, convert_type_typmod() takes care of it. + */ +static Datum +convert_via_io(Datum value, Oid sourceTypeId, Oid targetTypeId, + bool *is_failed) +{ + FmgrInfo outfunc; + Oid infuncId = InvalidOid; + Oid outfuncId = InvalidOid; + Oid intypioparam = InvalidOid; + bool outtypisvarlena = false; + MemoryContext oldcontext = CurrentMemoryContext; + volatile Datum res = (Datum) 0; + + /* Perhaps the types are domains; if so, look at their base types */ + sourceTypeId = getBaseType(sourceTypeId); + targetTypeId = getBaseType(targetTypeId); + + getTypeOutputInfo(sourceTypeId, &outfuncId, &outtypisvarlena); + fmgr_info(outfuncId, &outfunc); + + getTypeInputInfo(targetTypeId, &infuncId, &intypioparam); + + PG_TRY(); + { + char *string; + + /* the caller has already rejected a NULL input value */ + string = OutputFunctionCall(&outfunc, value); + + res = OidFunctionCall3(infuncId, + CStringGetDatum(string), + ObjectIdGetDatum(intypioparam), + Int32GetDatum(-1)); + + pfree(string); + } + PG_CATCH(); + { + report_conversion_error(oldcontext, is_failed); + } + PG_END_TRY(); + + return res; +} + +/* + * Get the actual typmod of a specific function argument (counting from 0), + * but working from the calling expression tree instead of FmgrInfo. + * + * Returns -1 if information is not available. + */ +static int32 +get_call_expr_argtypmod(Node *expr, int argnum) +{ + List *args; + + if (expr == NULL) + return -1; + + if (IsA(expr, FuncExpr)) + args = ((FuncExpr *) expr)->args; + else if (IsA(expr, OpExpr)) + args = ((OpExpr *) expr)->args; + else if (IsA(expr, DistinctExpr)) + args = ((DistinctExpr *) expr)->args; + else if (IsA(expr, ScalarArrayOpExpr)) + args = ((ScalarArrayOpExpr *) expr)->args; + else if (IsA(expr, ArrayCoerceExpr)) + args = list_make1(((ArrayCoerceExpr *) expr)->arg); + else if (IsA(expr, NullIfExpr)) + args = ((NullIfExpr *) expr)->args; + else if (IsA(expr, WindowFunc)) + args = ((WindowFunc *) expr)->args; + else + return -1; + + if (argnum < 0 || argnum >= list_length(args)) + return -1; + + return exprTypmod((Node *) list_nth(args, argnum)); +} + +/* + * Get the actual typmod of a specific function argument (counting from 0). + * + * Returns -1 if information is not available. + */ +static int32 +get_fn_expr_argtypmod(FmgrInfo *flinfo, int argnum) +{ + /* + * can't return anything useful if we have no FmgrInfo or if its fn_expr + * node has not been initialized + */ + if (!flinfo || !flinfo->fn_expr) + return -1; + + return get_call_expr_argtypmod(flinfo->fn_expr, argnum); +} + +/* + * Apply the conversion found by find_conversion_way() or + * find_typmod_conversion_function(). + * + * A conversion that cannot be performed at all is a query error and is + * reported as such; only failures caused by the data being converted are + * reported through *is_failed. + */ +static Datum +convert(Datum value, ConversionType conversion_type, Oid funcId, + Oid sourceTypeId, Oid targetTypeId, int32 targetTypMod, + bool *is_failed) +{ + switch (conversion_type) + { + case CONVERSION_TYPE_RELABEL: + return value; + + case CONVERSION_TYPE_FUNC: + return convert_from_function(value, targetTypMod, funcId, + is_failed); + + case CONVERSION_TYPE_VIA_IO: + return convert_via_io(value, sourceTypeId, targetTypeId, + is_failed); + + case CONVERSION_TYPE_ARRAY: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("try_convert() does not support casts between array types"), + errdetail("Cannot cast type %s to %s.", + format_type_be(sourceTypeId), + format_type_be(targetTypeId)))); + break; + + case CONVERSION_TYPE_NONE: + ereport(ERROR, + (errcode(ERRCODE_CANNOT_COERCE), + errmsg("cannot cast type %s to %s", + format_type_be(sourceTypeId), + format_type_be(targetTypeId)))); + break; + } + + elog(ERROR, "unrecognized conversion method: %d", (int) conversion_type); + return (Datum) 0; /* keep compiler quiet */ +} + +/* + * Coerce a value of targetTypeId to targetTypMod. + */ +static Datum +convert_type_typmod(Datum value, int32 sourceTypMod, Oid targetTypeId, + int32 targetTypMod, bool *is_failed) +{ + ConversionType conversion_type; + Oid funcId; + + if (targetTypMod < 0 || targetTypMod == sourceTypMod) + return value; + + conversion_type = find_typmod_conversion_function(targetTypeId, &funcId); + + /* + * If the target type has no length coercion function, just leave the value + * alone, the same way coerce_type_typmod() does. + */ + if (conversion_type == CONVERSION_TYPE_NONE) + return value; + + return convert(value, conversion_type, funcId, targetTypeId, targetTypeId, + targetTypMod, is_failed); +} + +/* + * try_convert(source_value, default_value) -> converted value or default_value + * + * The target type is taken from the second argument, which is also the value + * returned when the conversion fails. + */ +Datum +try_convert(PG_FUNCTION_ARGS) +{ + Oid sourceTypeId; + int32 sourceTypMod; + Oid targetTypeId; + int32 targetTypMod; + Oid baseTypeId; + int32 baseTypMod; + Oid funcId; + ConversionType conversion_type; + Datum value; + Datum res; + int32 resTypMod; + bool is_failed = false; + + /* A NULL input converts to NULL, whatever the default value is */ + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + + sourceTypeId = get_fn_expr_argtype(fcinfo->flinfo, 0); + sourceTypMod = get_fn_expr_argtypmod(fcinfo->flinfo, 0); + + targetTypeId = get_fn_expr_argtype(fcinfo->flinfo, 1); + targetTypMod = get_fn_expr_argtypmod(fcinfo->flinfo, 1); + + if (!OidIsValid(sourceTypeId) || !OidIsValid(targetTypeId)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not determine the argument types of try_convert()"))); + + baseTypMod = targetTypMod; + baseTypeId = getBaseTypeAndTypmod(targetTypeId, &baseTypMod); + + if (targetTypeId != baseTypeId) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("try_convert() does not support casts to domain types"), + errdetail("Cannot cast type %s to %s.", + format_type_be(sourceTypeId), + format_type_be(targetTypeId)))); + + value = PG_GETARG_DATUM(0); + + conversion_type = find_conversion_way(targetTypeId, sourceTypeId, &funcId); + + if (conversion_type == CONVERSION_TYPE_RELABEL) + { + res = value; + resTypMod = sourceTypMod; + } + else + { + res = convert(value, conversion_type, funcId, sourceTypeId, baseTypeId, + baseTypMod, &is_failed); + resTypMod = -1; + } + + if (!is_failed) + res = convert_type_typmod(res, resTypMod, targetTypeId, targetTypMod, + &is_failed); + + if (is_failed) + { + /* the value could not be converted, fall back to the default one */ + fcinfo->isnull = PG_ARGISNULL(1); + return PG_GETARG_DATUM(1); + } + + return res; +} diff --git a/contrib/try_convert/try_convert.control b/contrib/try_convert/try_convert.control new file mode 100644 index 00000000000..022ba632784 --- /dev/null +++ b/contrib/try_convert/try_convert.control @@ -0,0 +1,6 @@ +# try_convert extension +comment = 'function for type cast' +default_version = '1.0' +module_pathname = '$libdir/try_convert' +relocatable = true +trusted = true From e2e4b2b27bc7456c220cd6becf7371d00e7bde02 Mon Sep 17 00:00:00 2001 From: Vlasdislav Date: Thu, 13 Aug 2026 18:34:11 +0300 Subject: [PATCH 158/167] Fix(ci): checkout to REL_2_STABLE --- .asf.yaml | 16 ++-------------- .github/workflows/apache-rat-audit.yml | 4 ++-- .github/workflows/binary-swap-check.yml | 8 ++++---- .github/workflows/build-cloudberry-rocky8.yml | 4 ++-- .github/workflows/build-cloudberry.yml | 4 ++-- .github/workflows/build-dbg-cloudberry.yml | 4 ++-- .../build-deb-cloudberry-ubuntu24.04.yml | 4 ++-- .github/workflows/build-deb-cloudberry.yml | 4 ++-- .../workflows/docker-cbdb-build-containers.yml | 6 +++--- .../workflows/docker-cbdb-test-containers.yml | 6 +++--- .github/workflows/yezzey-ci.yaml | 6 +++--- 11 files changed, 27 insertions(+), 39 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 01188659355..3504f4b2a7a 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -72,9 +72,9 @@ github: # Enable rebase merging for linear history rebase: true - # Branch protection rules for the main branch + # Branch protection rules for the REL_2_STABLE release branch protected_branches: - main: + REL_2_STABLE: # Require status checks to pass before merging required_status_checks: # Require branches to be up to date before merging @@ -122,18 +122,6 @@ github: # Require conversation threads to be resolved required_conversation_resolution: true - # Branch protection for REL_2_STABLE release branch - REL_2_STABLE: - # Pull request review requirements - required_pull_request_reviews: - # Require new reviews when new commits are pushed - dismiss_stale_reviews: false - # Require at least 2 approving reviews - required_approving_review_count: 2 - - # Require conversation threads to be resolved - required_conversation_resolution: true - # Branch cleanup settings # Don't automatically delete branches after merging del_branch_on_merge: true diff --git a/.github/workflows/apache-rat-audit.yml b/.github/workflows/apache-rat-audit.yml index 4826fc89228..e14852bc317 100644 --- a/.github/workflows/apache-rat-audit.yml +++ b/.github/workflows/apache-rat-audit.yml @@ -32,9 +32,9 @@ name: Apache Rat License Check on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: diff --git a/.github/workflows/binary-swap-check.yml b/.github/workflows/binary-swap-check.yml index 3d3f5847eb7..6b21721fc57 100644 --- a/.github/workflows/binary-swap-check.yml +++ b/.github/workflows/binary-swap-check.yml @@ -171,7 +171,7 @@ jobs: - name: Checkout Build Tools uses: actions/checkout@v4 with: - ref: main + ref: REL_2_STABLE path: build_tools sparse-checkout: | devops @@ -181,7 +181,7 @@ jobs: run: | # Copy devops scripts to baseline source if they don't exist if [ ! -d "devops" ]; then - echo "Injecting devops scripts from main branch..." + echo "Injecting devops scripts from REL_2_STABLE branch..." cp -r build_tools/devops . fi @@ -316,7 +316,7 @@ jobs: - name: Checkout Build Tools uses: actions/checkout@v4 with: - ref: main + ref: REL_2_STABLE path: build_tools sparse-checkout: | devops @@ -326,7 +326,7 @@ jobs: run: | # Copy devops scripts to current source if they don't exist if [ ! -d "devops" ]; then - echo "Injecting devops scripts from main branch..." + echo "Injecting devops scripts from REL_2_STABLE branch..." cp -r build_tools/devops . fi diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index ceb2eb10950..11204ec35be 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -103,12 +103,12 @@ name: Apache Cloudberry Build (Rocky 8) on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: paths: - '.github/workflows/build-cloudberry-rocky8.yml' # We can enable the PR test when needed - # branches: [main, REL_2_STABLE] + # branches: [REL_2_STABLE] # types: [opened, synchronize, reopened, edited] schedule: # Run every Monday at 02:00 UTC diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 289592cd405..cf8bb13d2eb 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -102,9 +102,9 @@ name: Apache Cloudberry Build on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: inputs: diff --git a/.github/workflows/build-dbg-cloudberry.yml b/.github/workflows/build-dbg-cloudberry.yml index 967fc259f0b..62be28b7121 100644 --- a/.github/workflows/build-dbg-cloudberry.yml +++ b/.github/workflows/build-dbg-cloudberry.yml @@ -102,9 +102,9 @@ name: Apache Cloudberry Build Debug on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: inputs: diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml index 072a0e77258..6881da4a3a5 100644 --- a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml +++ b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml @@ -75,12 +75,12 @@ name: Apache Cloudberry Debian Build on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: paths: - '.github/workflows/build-deb-cloudberry-ubuntu24.04.yml' # We can enable the PR test when needed - # branches: [main, REL_2_STABLE] + # branches: [REL_2_STABLE] # types: [opened, synchronize, reopened, edited] schedule: # Run every Monday at 02:00 UTC diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index 3c6b2145719..a054ed67e5c 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -74,9 +74,9 @@ name: Apache Cloudberry Debian Build on: push: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] pull_request: - branches: [main, REL_2_STABLE] + branches: [REL_2_STABLE] types: [opened, synchronize, reopened, edited] workflow_dispatch: # Manual trigger inputs: diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 538b4e9b179..2e5c42ad93a 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -125,7 +125,7 @@ jobs: # Login to DockerHub for pushing images # Requires DOCKERHUB_USER and DOCKERHUB_TOKEN secrets to be set - name: Login to Docker Hub - if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/REL_2_STABLE' }} uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} @@ -174,7 +174,7 @@ jobs: # Build and push multi-architecture images # This creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images - if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/REL_2_STABLE' }} uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/build/${{ matrix.platform }} @@ -204,7 +204,7 @@ jobs: echo "- ✅ Dockerfile syntax validated" >> $GITHUB_STEP_SUMMARY echo "- ✅ Multi-architecture builds tested" >> $GITHUB_STEP_SUMMARY echo "- ✅ TestInfra tests executed" >> $GITHUB_STEP_SUMMARY - echo "- ⏭️ Docker Hub push skipped (requires main branch)" >> $GITHUB_STEP_SUMMARY + echo "- ⏭️ Docker Hub push skipped (requires REL_2_STABLE branch)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY fi diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 4d0fb8def33..27cf9ec063e 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -113,7 +113,7 @@ jobs: # Login to DockerHub for pushing images - name: Login to Docker Hub - if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/REL_2_STABLE' }} uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.DOCKERHUB_USER }} @@ -144,7 +144,7 @@ jobs: # Build and push multi-architecture images # Creates a manifest list that supports both architectures - name: Build and Push Multi-arch Docker images - if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ steps.platform-filter.outputs[matrix.platform] == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/REL_2_STABLE' }} uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./devops/deploy/docker/test/${{ matrix.platform }} @@ -177,7 +177,7 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "- ✅ Dockerfile syntax validated" >> $GITHUB_STEP_SUMMARY echo "- ✅ Multi-architecture builds tested" >> $GITHUB_STEP_SUMMARY - echo "- ⏭️ Docker Hub push skipped (requires main branch)" >> $GITHUB_STEP_SUMMARY + echo "- ⏭️ Docker Hub push skipped (requires REL_2_STABLE branch)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY fi diff --git a/.github/workflows/yezzey-ci.yaml b/.github/workflows/yezzey-ci.yaml index 82511c4f701..bc21ceef616 100644 --- a/.github/workflows/yezzey-ci.yaml +++ b/.github/workflows/yezzey-ci.yaml @@ -23,7 +23,7 @@ name: Yezzey CI Pipeline on: push: - branches: [ main ] + branches: [ REL_2_STABLE ] pull_request: types: [opened, synchronize, reopened, edited] workflow_dispatch: @@ -33,11 +33,11 @@ permissions: concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/REL_2_STABLE' }} env: CLOUDBERRY_HOME: "/usr/local/cloudberry-db" - CLOUDBERRY_VERSION: "main" + CLOUDBERRY_VERSION: "REL_2_STABLE" jobs: From 315ba11d1f6fb142b474a110b0c813d4649b4308 Mon Sep 17 00:00:00 2001 From: Vlasdislav Date: Fri, 14 Aug 2026 14:27:34 +0300 Subject: [PATCH 159/167] Fix(ci): Resolve binary swap baseline from open-gpdb tags --- .github/workflows/binary-swap-check.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/binary-swap-check.yml b/.github/workflows/binary-swap-check.yml index 6b21721fc57..25d51ce0576 100644 --- a/.github/workflows/binary-swap-check.yml +++ b/.github/workflows/binary-swap-check.yml @@ -104,8 +104,9 @@ jobs: # Exclude RC versions: X.Y.Z-incubating-rcN echo "Finding latest release tag..." - # Get all tags matching the incubating pattern, sort by version, filter out RC versions - TAG_INFO=$(git ls-remote --tags --refs origin '*-incubating' 2>/dev/null \ + # Get all tags matching the incubating pattern from open-gpdb, + # sort by version, filter out RC versions. + TAG_INFO=$(git ls-remote --tags --refs https://github.com/open-gpdb/cloudberry.git '*-incubating' 2>/dev/null \ | grep -v '\-rc[0-9]*$' \ | sort -t'/' -k3 -V \ | tail -1 || echo "") @@ -164,6 +165,7 @@ jobs: - name: Checkout Baseline Code uses: actions/checkout@v4 with: + repository: open-gpdb/cloudberry ref: ${{ needs.resolve-baseline.outputs.baseline_version }} fetch-depth: 1 submodules: recursive From 63c85ab5683ab6c204af1954aca884ead2eed831 Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:49:27 +0300 Subject: [PATCH 160/167] Update binary swap systables expected output (#56) --- .../upgrading_compatibility/systables.out | 191 +++++++----------- 1 file changed, 70 insertions(+), 121 deletions(-) diff --git a/src/test/binary_swap/expected/upgrading_compatibility/systables.out b/src/test/binary_swap/expected/upgrading_compatibility/systables.out index ef58176f480..fdf47a70c90 100644 --- a/src/test/binary_swap/expected/upgrading_compatibility/systables.out +++ b/src/test/binary_swap/expected/upgrading_compatibility/systables.out @@ -132,7 +132,6 @@ Indexes: View "pg_catalog.pg_stat_progress_copy" Column | Type | Collation | Nullable | Default ------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | datid | oid | | | datname | name | | | @@ -148,7 +147,6 @@ Indexes: View "pg_catalog.pg_stat_progress_basebackup" Column | Type | Collation | Nullable | Default ----------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | phase | text | | | backup_total | bigint | | | @@ -160,7 +158,6 @@ Indexes: View "pg_catalog.pg_stat_progress_create_index" Column | Type | Collation | Nullable | Default --------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | datid | oid | | | datname | name | | | @@ -182,7 +179,6 @@ Indexes: View "pg_catalog.pg_stat_progress_cluster" Column | Type | Collation | Nullable | Default ---------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | datid | oid | | | datname | name | | | @@ -200,7 +196,6 @@ Indexes: View "pg_catalog.pg_stat_progress_vacuum" Column | Type | Collation | Nullable | Default --------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | datid | oid | | | datname | name | | | @@ -217,7 +212,6 @@ Indexes: View "pg_catalog.pg_stat_progress_analyze" Column | Type | Collation | Nullable | Default ---------------------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | pid | integer | | | datid | oid | | | datname | name | | | @@ -235,9 +229,8 @@ Indexes: View "pg_catalog.pg_stat_wal" Column | Type | Collation | Nullable | Default ------------------+--------------------------+-----------+----------+--------- - gp_segment_id | integer | | | wal_records | bigint | | | - wal_fpi | bigint | | | + wal_fpw | bigint | | | wal_bytes | numeric | | | wal_buffers_full | bigint | | | wal_write | bigint | | | @@ -407,22 +400,20 @@ Indexes: rsqholders | integer | | | \d pg_stat_resqueues - View "pg_catalog.pg_stat_resqueues" - Column | Type | Collation | Nullable | Default -----------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | - queueid | oid | | | - queuename | name | | | - n_queries_exec | bigint | | | - n_queries_wait | bigint | | | - elapsed_exec | bigint | | | - elapsed_wait | bigint | | | + View "pg_catalog.pg_stat_resqueues" + Column | Type | Collation | Nullable | Default +----------------+--------+-----------+----------+--------- + queueid | oid | | | + queuename | name | | | + n_queries_exec | bigint | | | + n_queries_wait | bigint | | | + elapsed_exec | bigint | | | + elapsed_wait | bigint | | | \d pg_stat_database View "pg_catalog.pg_stat_database" Column | Type | Collation | Nullable | Default --------------------------+--------------------------+-----------+----------+--------- - gp_segment_id | integer | | | datid | oid | | | datname | name | | | numbackends | integer | | | @@ -453,20 +444,19 @@ Indexes: stats_reset | timestamp with time zone | | | \d pg_stat_replication_slots - View "pg_catalog.pg_stat_replication_slots" - Column | Type | Collation | Nullable | Default ----------------+--------------------------+-----------+----------+--------- - gp_segment_id | integer | | | - slot_name | text | | | - spill_txns | bigint | | | - spill_count | bigint | | | - spill_bytes | bigint | | | - stream_txns | bigint | | | - stream_count | bigint | | | - stream_bytes | bigint | | | - total_txns | bigint | | | - total_bytes | bigint | | | - stats_reset | timestamp with time zone | | | + View "pg_catalog.pg_stat_replication_slots" + Column | Type | Collation | Nullable | Default +--------------+--------------------------+-----------+----------+--------- + slot_name | text | | | + spill_txns | bigint | | | + spill_count | bigint | | | + spill_bytes | bigint | | | + stream_txns | bigint | | | + stream_count | bigint | | | + stream_bytes | bigint | | | + total_txns | bigint | | | + total_bytes | bigint | | | + stats_reset | timestamp with time zone | | | \d pg_replication_slots View "pg_catalog.pg_replication_slots" @@ -798,46 +788,33 @@ Indexes: idx_blks_hit | bigint | | | \d pg_stat_user_indexes - View "pg_catalog.pg_stat_user_indexes" - Column | Type | Collation | Nullable | Default ----------------+---------+-----------+----------+--------- - relid | oid | | | - indexrelid | oid | | | - schemaname | name | | | - relname | name | | | - indexrelname | name | | | - idx_scan | numeric | | | - idx_tup_read | numeric | | | - idx_tup_fetch | numeric | | | + View "pg_catalog.pg_stat_user_indexes" + Column | Type | Collation | Nullable | Default +---------------+--------+-----------+----------+--------- + relid | oid | | | + indexrelid | oid | | | + schemaname | name | | | + relname | name | | | + indexrelname | name | | | + idx_scan | bigint | | | + idx_tup_read | bigint | | | + idx_tup_fetch | bigint | | | \d pg_stat_sys_indexes View "pg_catalog.pg_stat_sys_indexes" - Column | Type | Collation | Nullable | Default ----------------+---------+-----------+----------+--------- - relid | oid | | | - indexrelid | oid | | | - schemaname | name | | | - relname | name | | | - indexrelname | name | | | - idx_scan | numeric | | | - idx_tup_read | numeric | | | - idx_tup_fetch | numeric | | | + Column | Type | Collation | Nullable | Default +---------------+--------+-----------+----------+--------- + relid | oid | | | + indexrelid | oid | | | + schemaname | name | | | + relname | name | | | + indexrelname | name | | | + idx_scan | bigint | | | + idx_tup_read | bigint | | | + idx_tup_fetch | bigint | | | \d pg_stat_all_indexes View "pg_catalog.pg_stat_all_indexes" - Column | Type | Collation | Nullable | Default ----------------+---------+-----------+----------+--------- - relid | oid | | | - indexrelid | oid | | | - schemaname | name | | | - relname | name | | | - indexrelname | name | | | - idx_scan | numeric | | | - idx_tup_read | numeric | | | - idx_tup_fetch | numeric | | | - -\d pg_stat_all_indexes_internal - View "pg_catalog.pg_stat_all_indexes_internal" Column | Type | Collation | Nullable | Default ---------------+--------+-----------+----------+--------- relid | oid | | | @@ -849,6 +826,7 @@ Indexes: idx_tup_read | bigint | | | idx_tup_fetch | bigint | | | +\d pg_stat_all_indexes_internal \d pg_statio_user_tables View "pg_catalog.pg_statio_user_tables" Column | Type | Collation | Nullable | Default @@ -948,18 +926,18 @@ Indexes: relid | oid | | | schemaname | name | | | relname | name | | | - seq_scan | numeric | | | - seq_tup_read | numeric | | | - idx_scan | numeric | | | - idx_tup_fetch | numeric | | | - n_tup_ins | numeric | | | - n_tup_upd | numeric | | | - n_tup_del | numeric | | | - n_tup_hot_upd | numeric | | | - n_live_tup | numeric | | | - n_dead_tup | numeric | | | - n_mod_since_analyze | numeric | | | - n_ins_since_vacuum | numeric | | | + seq_scan | bigint | | | + seq_tup_read | bigint | | | + idx_scan | bigint | | | + idx_tup_fetch | bigint | | | + n_tup_ins | bigint | | | + n_tup_upd | bigint | | | + n_tup_del | bigint | | | + n_tup_hot_upd | bigint | | | + n_live_tup | bigint | | | + n_dead_tup | bigint | | | + n_mod_since_analyze | bigint | | | + n_ins_since_vacuum | bigint | | | last_vacuum | timestamp with time zone | | | last_autovacuum | timestamp with time zone | | | last_analyze | timestamp with time zone | | | @@ -1020,18 +998,18 @@ Indexes: relid | oid | | | schemaname | name | | | relname | name | | | - seq_scan | numeric | | | - seq_tup_read | numeric | | | - idx_scan | numeric | | | - idx_tup_fetch | numeric | | | - n_tup_ins | numeric | | | - n_tup_upd | numeric | | | - n_tup_del | numeric | | | - n_tup_hot_upd | numeric | | | - n_live_tup | numeric | | | - n_dead_tup | numeric | | | - n_mod_since_analyze | numeric | | | - n_ins_since_vacuum | numeric | | | + seq_scan | bigint | | | + seq_tup_read | bigint | | | + idx_scan | bigint | | | + idx_tup_fetch | bigint | | | + n_tup_ins | bigint | | | + n_tup_upd | bigint | | | + n_tup_del | bigint | | | + n_tup_hot_upd | bigint | | | + n_live_tup | bigint | | | + n_dead_tup | bigint | | | + n_mod_since_analyze | bigint | | | + n_ins_since_vacuum | bigint | | | last_vacuum | timestamp with time zone | | | last_autovacuum | timestamp with time zone | | | last_analyze | timestamp with time zone | | | @@ -1060,34 +1038,6 @@ Indexes: \d pg_stat_all_tables View "pg_catalog.pg_stat_all_tables" Column | Type | Collation | Nullable | Default ----------------------+--------------------------+-----------+----------+--------- - relid | oid | | | - schemaname | name | | | - relname | name | | | - seq_scan | numeric | | | - seq_tup_read | numeric | | | - idx_scan | numeric | | | - idx_tup_fetch | numeric | | | - n_tup_ins | numeric | | | - n_tup_upd | numeric | | | - n_tup_del | numeric | | | - n_tup_hot_upd | numeric | | | - n_live_tup | numeric | | | - n_dead_tup | numeric | | | - n_mod_since_analyze | numeric | | | - n_ins_since_vacuum | numeric | | | - last_vacuum | timestamp with time zone | | | - last_autovacuum | timestamp with time zone | | | - last_analyze | timestamp with time zone | | | - last_autoanalyze | timestamp with time zone | | | - vacuum_count | bigint | | | - autovacuum_count | bigint | | | - analyze_count | bigint | | | - autoanalyze_count | bigint | | | - -\d pg_stat_all_tables_internal - View "pg_catalog.pg_stat_all_tables_internal" - Column | Type | Collation | Nullable | Default ---------------------+--------------------------+-----------+----------+--------- relid | oid | | | schemaname | name | | | @@ -1113,6 +1063,7 @@ Indexes: analyze_count | bigint | | | autoanalyze_count | bigint | | | +\d pg_stat_all_tables_internal \d pg_timezone_names View "pg_catalog.pg_timezone_names" Column | Type | Collation | Nullable | Default @@ -1147,7 +1098,6 @@ Indexes: View "pg_catalog.pg_prepared_statements" Column | Type | Collation | Nullable | Default -----------------+--------------------------+-----------+----------+--------- - gp_segment_id | integer | | | name | text | | | statement | text | | | prepare_time | timestamp with time zone | | | @@ -2889,7 +2839,6 @@ Tablespace: "pg_global" View "pg_catalog.pg_backend_memory_contexts" Column | Type | Collation | Nullable | Default ---------------+---------+-----------+----------+--------- - gp_segment_id | integer | | | name | text | | | ident | text | | | parent | text | | | From 42e3c5a34143e230557ed6c6ef41a26ae4bd0471 Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:50:03 +0300 Subject: [PATCH 161/167] Import uuid_cb into gpcontrib from Greenplum (#55) * Initial commit * Fix string out of bounds error and change functions to use text type * Add readme * Add debian packaging configuration * Feat: Adapt uuid_cb for Cloudberry * Refactor: Minor refactoring. Delete Windows adapt. Update README.md and comments --------- Co-authored-by: Maxim Smyatkin --- .github/workflows/build-cloudberry-rocky8.yml | 3 +- .github/workflows/build-cloudberry.yml | 3 +- gpcontrib/Makefile | 6 +- gpcontrib/uuid_cb/.gitignore | 5 + gpcontrib/uuid_cb/Makefile | 20 +++ gpcontrib/uuid_cb/README.md | 71 +++++++++ gpcontrib/uuid_cb/sql/uuid-cb--1.0.sql | 14 ++ .../uuid_cb/sql/uuid-cb--unpackaged--1.0.sql | 7 + gpcontrib/uuid_cb/src/uid.c | 94 +++++++++++ gpcontrib/uuid_cb/src/uid.h | 37 +++++ gpcontrib/uuid_cb/src/uuid-cb.c | 85 ++++++++++ gpcontrib/uuid_cb/src/uuid.c | 150 ++++++++++++++++++ gpcontrib/uuid_cb/src/uuid.h | 48 ++++++ gpcontrib/uuid_cb/test/expected/uuid_cb.out | 63 ++++++++ gpcontrib/uuid_cb/test/sql/uuid_cb.sql | 23 +++ gpcontrib/uuid_cb/uuid-cb.control | 6 + pom.xml | 7 + 17 files changed, 638 insertions(+), 4 deletions(-) create mode 100644 gpcontrib/uuid_cb/.gitignore create mode 100644 gpcontrib/uuid_cb/Makefile create mode 100644 gpcontrib/uuid_cb/README.md create mode 100644 gpcontrib/uuid_cb/sql/uuid-cb--1.0.sql create mode 100644 gpcontrib/uuid_cb/sql/uuid-cb--unpackaged--1.0.sql create mode 100644 gpcontrib/uuid_cb/src/uid.c create mode 100644 gpcontrib/uuid_cb/src/uid.h create mode 100644 gpcontrib/uuid_cb/src/uuid-cb.c create mode 100644 gpcontrib/uuid_cb/src/uuid.c create mode 100644 gpcontrib/uuid_cb/src/uuid.h create mode 100644 gpcontrib/uuid_cb/test/expected/uuid_cb.out create mode 100644 gpcontrib/uuid_cb/test/sql/uuid_cb.sql create mode 100644 gpcontrib/uuid_cb/uuid-cb.control diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 11204ec35be..bc0c8146a6d 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -321,7 +321,8 @@ jobs: "gpcontrib/zstd:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_toolkit:installcheck", - "gpcontrib/gp_url_tools:installcheck"] + "gpcontrib/gp_url_tools:installcheck", + "gpcontrib/uuid_cb:installcheck"] }, {"test":"gpcontrib-gp-stats-collector", "make_configs":["gpcontrib/gp_stats_collector:installcheck"], diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index cf8bb13d2eb..c20481da7b1 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -321,7 +321,8 @@ jobs: "gpcontrib/gp_toolkit:installcheck", "gpcontrib/gp_exttable_fdw:installcheck", "gpcontrib/gp_internal_tools:installcheck", - "gpcontrib/gp_url_tools:installcheck"] + "gpcontrib/gp_url_tools:installcheck", + "gpcontrib/uuid_cb:installcheck"] }, {"test":"ic-diskquota", "make_configs":["gpcontrib/diskquota:installcheck"], diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 0ee953d9904..fa1115a1cb5 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -23,7 +23,8 @@ ifeq "$(enable_debug_extensions)" "yes" gp_replica_check \ gp_toolkit \ gp_url_tools \ - pg_hint_plan + pg_hint_plan \ + uuid_cb else recurse_targets = gp_sparse_vector \ gp_distribution_policy \ @@ -32,7 +33,8 @@ else gp_exttable_fdw \ gp_toolkit \ gp_url_tools \ - pg_hint_plan + pg_hint_plan \ + uuid_cb endif ifeq "$(with_diskquota)" "yes" diff --git a/gpcontrib/uuid_cb/.gitignore b/gpcontrib/uuid_cb/.gitignore new file mode 100644 index 00000000000..3063aa777c5 --- /dev/null +++ b/gpcontrib/uuid_cb/.gitignore @@ -0,0 +1,5 @@ +/log/ +/results/ +/tmp_check/ +*.o +*.so diff --git a/gpcontrib/uuid_cb/Makefile b/gpcontrib/uuid_cb/Makefile new file mode 100644 index 00000000000..bd5ea4d0b3c --- /dev/null +++ b/gpcontrib/uuid_cb/Makefile @@ -0,0 +1,20 @@ +EXTENSION = uuid-cb +EXTVERSION = $(shell grep default_version $(EXTENSION).control | \ + sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") + +DATA = $(wildcard sql/*--*.sql) +REGRESS = uuid_cb +REGRESS_OPTS = --inputdir=test/ +OBJS = src/uuid-cb.o src/uuid.o src/uid.o + +MODULE_big = uuid-cb + +ifdef USE_PGXS + PG_CONFIG = pg_config + PGXS := $(shell $(PG_CONFIG) --pgxs) + include $(PGXS) +else + top_builddir = ../.. + include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/gpcontrib/uuid_cb/README.md b/gpcontrib/uuid_cb/README.md new file mode 100644 index 00000000000..bedc89ecfee --- /dev/null +++ b/gpcontrib/uuid_cb/README.md @@ -0,0 +1,71 @@ + + +## Overview +UUID-CB PostgreSQL/Greenplum/Cloudberry extension provides functions to generate text UUIDs in format required by Russian Central Bank. The only difference from commonly used UUID format is that CB UUIDs have additional hexadecimal character for checksum: + + UUID: f6553a80-642d-11ed-854e-09a55775d327 + UUID-CB: f6553a80-642d-11ed-854e-09a55775d327-9 + +Here "9" is the checksum. + +## Building +To build the extension you need those already installed: + +* GNU make +* GCC +* PostgreSQL/Greenplum/Cloudberry + +In addition you need to include greenplum/cloudberry binaries into PATH variable: + +``` +PATH="${GPHOME}/bin:${PATH}" +``` +This will allow build and install instructions to find pg_config, which provides the rest of information, necessary for the build: library paths, include paths, etc. + +Once it is done you can proceed with `make && make install` and it should install uuid-cb into proper location for your PostgreSQL/Greenplum/Cloudberry installation. + +### Running regression tests +To make sure the extension works well, you can start your database server: + +1. Set environment variables so that tests can find the server: `export PGPORT=your gp port` +2. Run `make installcheck` from uuid-cb root directory + +If everything is configured correctly, you should see one test passing. + +## Usage +To use the extension first, you need to install it as described above. Then you need to load the extension if it not loaded yet: + +```sql +CREATE EXTENSION IF NOT EXISTS "uuid-cb"; +``` + +After it`s done you can use it like this: + +```sql +SELECT uuid_cb_generate(); -- to generate a uuid +SELECT uuid_cb_valid(uuid_cb_generate()); -- to make sure the string is a valid CB UUID +``` +To create a table that uses CB UUIDs it is recommended to make the uuid column a primary key and distribute the table by it. Also it is recommended to add a check constraint on this column using `uuid_cb_valid`: + +```sql +CREATE TABLE uid_tbl (uid CHAR(38) NOT NULL DEFAULT uuid_cb_generate() \ +CHECK (uuid_cb_valid(uid) = true), data INTEGER NOT NULL, \ +PRIMARY KEY (uid)) DISTRIBUTED BY(uid); +``` \ No newline at end of file diff --git a/gpcontrib/uuid_cb/sql/uuid-cb--1.0.sql b/gpcontrib/uuid_cb/sql/uuid-cb--1.0.sql new file mode 100644 index 00000000000..6cedc15a0de --- /dev/null +++ b/gpcontrib/uuid_cb/sql/uuid-cb--1.0.sql @@ -0,0 +1,14 @@ +/* contrib/uuid-cb/uuid-cb--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "uuid-cb"''' to load this file. \quit + +CREATE FUNCTION uuid_cb_generate() +RETURNS text +AS 'MODULE_PATHNAME', 'uuid_cb_generate' +VOLATILE STRICT LANGUAGE C; + +CREATE FUNCTION uuid_cb_valid(text) +RETURNS boolean +AS 'MODULE_PATHNAME', 'uuid_cb_valid' +IMMUTABLE STRICT LANGUAGE C; diff --git a/gpcontrib/uuid_cb/sql/uuid-cb--unpackaged--1.0.sql b/gpcontrib/uuid_cb/sql/uuid-cb--unpackaged--1.0.sql new file mode 100644 index 00000000000..70c03722757 --- /dev/null +++ b/gpcontrib/uuid_cb/sql/uuid-cb--unpackaged--1.0.sql @@ -0,0 +1,7 @@ +/* contrib/uuid-cb/uuid-cb--unpackaged--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use '''CREATE EXTENSION "uuid-cb" FROM unpackaged''' to load this file. \quit + +ALTER EXTENSION "uuid-cb" ADD function uuid_cb_generate(); +ALTER EXTENSION "uuid-cb" ADD function uuid_cb_valid(text); diff --git a/gpcontrib/uuid_cb/src/uid.c b/gpcontrib/uuid_cb/src/uid.c new file mode 100644 index 00000000000..ccf4aa7201c --- /dev/null +++ b/gpcontrib/uuid_cb/src/uid.c @@ -0,0 +1,94 @@ +/* The original file is taken from https://www.cbr.ru/ckki/assignment_unique_id/ */ + +/* =================================================================== * + * Вариант реализации функции генерации уникального идентификатора * + * договора (сделки) в соответствии с указанием Банка России * + * "О правилах присвоения уникального идентификатора договора (сделки),* + * по обязательствам из которого (из которой) формируется кредитная * + * история" * + * =================================================================== */ + +#include +#include "uid.h" +#include "uuid.h" + +// Инициализация модуля +bool uid_init(void) { + return uuid_init(); +} + +// Деинициализация модуля (освобождение ресурсов) +void uid_deinit(void) { + uuid_deinit(); +} + +// Расчет контрольного символа +// str - указатель на строку, содержащую первую часть УИД. +char calc_ctrl(char *str) { + int pos = 0; // Позиция символа первой части УИД + int index = 1; // Индекс цифры первой части УИД + long sum = 0; // Сумма произведений цифр и их индексов + + // Цикл по строке до достижения конца строки + while (str[pos]) { + // Получение очередного символа строки + char c = str[pos++]; + + // Если символ - десятичная цифра + if (c >= '0' && c <= '9') { + // Увеличение суммы и индекса + sum += (c - '0') * (index++); + } + + // Если символ - шестнадцатеричная цифра + if (c >= 'a' && c <= 'f') { + // Увеличение суммы и индекса + sum += (c - 'a' + 10) * (index++); + } + + // Если значение индекса превысило 10, сбрасываем индекс в 1 + if (index > 10) { + index = 1; + } + } + + // Получение остатка от деления суммы произведений цифр с их индексом на 16 + int r = sum % 16; + + // Возврат результата в виде контрольного символа + if (r < 10) { + return (char)(r + '0'); + } else { + return (char)(r + 'a' - 10); + } +} + +// Создание УИД +// buffer - указатель на буфер длиной не менее 39 байт (38 байт УИД и 1 байт символ конца строки) +// Возвращаемый результат: +// - true - если УИД создан успешно и помещен в буфер +// - false - если УИД создать не удалось +bool uid_create(char *buffer) { + uuid_t uuid; + // Создание первой части УИД - УУИд + // Если УУИД создать не удалось, возвращаем отрицательный результат + if (!uuid_create(&uuid)) { + return false; + } + + // Преобразуем созданный УУИд в шестнадцатеричное представление + sprintf(buffer, "%8.8x-%4.4x-%4.4x-%2.2x%2.2x-%2.2x%2.2x%2.2x%2.2x%2.2x%2.2x-", + uuid.time_low, + uuid.time_mid, + uuid.time_hi_and_version, + uuid.clock_seq_hi_and_reserved, + uuid.clock_seq_low, + uuid.node[5], uuid.node[4], uuid.node[3], uuid.node[2], uuid.node[1], uuid.node[0]); + + // Расчет и помещение в буфер контрольного символа + buffer[UID_BUFFER_SIZE - 2] = calc_ctrl(buffer); + // Помещение в буфер символа конца строки + buffer[UID_BUFFER_SIZE - 1] = 0; + + return true; +} diff --git a/gpcontrib/uuid_cb/src/uid.h b/gpcontrib/uuid_cb/src/uid.h new file mode 100644 index 00000000000..0cae4ed8616 --- /dev/null +++ b/gpcontrib/uuid_cb/src/uid.h @@ -0,0 +1,37 @@ +/* The original file is taken from https://www.cbr.ru/ckki/assignment_unique_id/ */ + +/* =================================================================== * + * Вариант реализации функции генерации уникального идентификатора * + * договора (сделки) в соответствии с указанием Банка России * + * "О правилах присвоения уникального идентификатора договора (сделки),* + * по обязательствам из которого (из которой) формируется кредитная * + * история" * + * =================================================================== */ + +#include + +#define UID_BUFFER_SIZE 39 + +// Инициализация модуля генерации УИД. +// Данную функцию необходимо вызвать один раз перед первой генерацией УИД +// Возвращаемый результат: +// - true - инициализация выполнена успешно +// - false - не удалось выполнить инициализацию +// Если инициализация не выполнена, функция uid_create() будет завершаться +// ошибкой или возвращать некорректный результат +bool uid_init(void); + +// Деинициализация модуля УИД (освобождение ресурсов). +// Данную функция необходимо вызвать перед завершением работы приложения +// для освобождения ресурсов, занятых модулем. +void uid_deinit(void); + +// Создание УИД +// buffer - указатель на буфер длиной не менее 39 байт (38 байт УИД и 1 байт символ конца строки) +// Возвращаемый результат: +// - true - если УИД создан успешно и помещен в буфер +// - false - если УИД создать не удалось +bool uid_create(char *buffer); + +// Вычисление контрольного символа по УИД +char calc_ctrl(char *str); diff --git a/gpcontrib/uuid_cb/src/uuid-cb.c b/gpcontrib/uuid_cb/src/uuid-cb.c new file mode 100644 index 00000000000..275ae77ef85 --- /dev/null +++ b/gpcontrib/uuid_cb/src/uuid-cb.c @@ -0,0 +1,85 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * uuid-cb.c + * + * IDENTIFICATION + * gpcontrib/uuid_cb/src/uuid-cb.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "uid.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(uuid_cb_generate); +PG_FUNCTION_INFO_V1(uuid_cb_valid); + +void _PG_init(void); +void _PG_fini(void); + +#define UUID_LEN 36 +#define CB_UUID_LEN 38 + +Datum +uuid_cb_generate(PG_FUNCTION_ARGS) +{ + char buf[CB_UUID_LEN + 1]; + if (!uid_create(buf)) { + ereport(ERROR, + (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), + errmsg("Could not generate CB UUID"))); + } + PG_RETURN_TEXT_P(cstring_to_text_with_len(buf, CB_UUID_LEN)); +} + +Datum +uuid_cb_valid(PG_FUNCTION_ARGS) +{ + char *uuid_cb_str; + char uuid_str[UUID_LEN + 1]; + uuid_cb_str = text_to_cstring(PG_GETARG_TEXT_PP(0)); + if (!uuid_cb_str || strlen(uuid_cb_str) != CB_UUID_LEN) { + PG_RETURN_BOOL(false); + } + strncpy(uuid_str, uuid_cb_str, UUID_LEN); + uuid_str[UUID_LEN] = 0; + PG_RETURN_BOOL(calc_ctrl(uuid_str) == uuid_cb_str[CB_UUID_LEN - 1]); +} + +void +_PG_init(void) +{ + if (!uid_init()) { + ereport(ERROR, + (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), + errmsg("Could not initialize CB UUID generator"))); + } +} + +void +_PG_fini(void) +{ + uid_deinit(); +} \ No newline at end of file diff --git a/gpcontrib/uuid_cb/src/uuid.c b/gpcontrib/uuid_cb/src/uuid.c new file mode 100644 index 00000000000..1b8be77a2f7 --- /dev/null +++ b/gpcontrib/uuid_cb/src/uuid.c @@ -0,0 +1,150 @@ +/* The original file is taken from https://www.cbr.ru/ckki/assignment_unique_id/ */ + +/* =================================================================== * + * Вариант реализации функции генерации первой части уникального * + * идентификатора договора (сделки) - универсального уникального * + * идентификатора в соответствии с указанием Банка России * + * "О правилах присвоения уникального идентификатора договора (сделки),* + * по обязательствам из которого (из которой) формируется кредитная * + * история" * + * =================================================================== */ + +#include +#include +#include +#include +int hRandom = -1; + +#include "uuid.h" + +bool generator_init(void); +bool GetRND(uint32_t *); +uint64_t GetTime(void); + + +// Инициализация +bool uuid_init(void) { + hRandom = open("/dev/random", O_RDONLY); + if (hRandom < 0) { + return false; + } + + // Инициализация платформо-независимой части + return generator_init(); +} + +// Деинициализация (освобождение ресурсов) +void uuid_deinit(void) { + close(hRandom); + hRandom = -1; +} + +// Получение 32-битного случайного числа +bool GetRND(uint32_t *rnd) { + if (read(hRandom, rnd, sizeof(uint32_t)) < 0) { + return false; + } + return true; +} + +// Получение времени в 100-наносекундных интервалах от 15.10.1580 +uint64_t GetTime(void) { + struct timeval tp; + + if (gettimeofday(&tp, (struct timezone *)0) != 0) { + return 0; + } + + // Приводим время к 100-наносекундным интервалам от 15.10.1580 + return ((uint64_t)tp.tv_sec * 10000000) + ((uint64_t)(tp.tv_usec / 1000) * 10000) + 0x01B21DD213814000; +} + +// Глобальные переменные модуля +uint8_t gNode[6]; // Узел +uint16_t gClockSeq; // Поле Clock sequence +uint64_t gLastTime = 0; // Время генерации последнего УУИд +int gLastUSNS = 0; // Значение микро и наносекунд при генерации последнего УУИд + +// Инициализация +bool generator_init(void) { + uint32_t rnd; + + // Получаем значение поля Node - как случайное число + if (!GetRND(&rnd)) { + return false; + } + memmove(&gNode[0], &rnd, 4); + if (!GetRND(&rnd)) { + return false; + } + memmove(&gNode[4], &rnd, 2); + gNode[5] |= 0x01; + + // Получаем значение поля Clock sequence + if (!GetRND(&rnd)) { + return false; + } + gClockSeq = rnd & 0x1FFF; + + return true; +} + +// Генерация УУИд +// uuid - указатель на структуру uuid_t для помещения в нее данные УУИд +// Возвращаемый результат: +// - true - если УУИд создан успешно и помещен в структуру +// - false - если УУИд создать не удалось +bool uuid_create(uuid_t *uuid) +{ + uint64_t time; + + // Получение времени + // + // В течении 1 миллисекунды может быть сгенерировано + // максимум 10000 уникальных УУИд. + // Если в текущую миллисекунду уже сгенерировано 10000 УУИд, + // будем ждать следующую миллисекунду. + do { + time = GetTime(); + if (!time) { + return false; + } + } while (time == gLastTime && gLastUSNS == 9999); + + if (time == gLastTime) { + // Если время не изменилось, за микро- и наносекунды + // возьмем предыдущее значение плюс один + gLastUSNS++; + } else { + // Если время изменилось, за микро- и наносекунды возьмём 0. + gLastUSNS = 0; + // и запомним значение времени + gLastTime = time; + } + + // Прибавим к значению времени (полученного с точностью до миллисекунд) + // выбранное выше значение микро- и наносекунд + time += gLastUSNS; + + // Заполним поля УУИд + // Младшая часть времени + uuid->time_low = (uint32_t)(time & 0xFFFFFFFF); + // Средняя часть времени + uuid->time_mid = (uint16_t)((time >> 32) & 0xFFFF); + // Старшая часть времени + uuid->time_hi_and_version = (uint16_t)((time >> 48) & 0x0FFF); + // Версия + uuid->time_hi_and_version |= (1 << 12); + + // Младшая часть временной последовательности + uuid->clock_seq_low = (uint8_t)(gClockSeq & 0xFF); + // Старшая часть временной последовательности + uuid->clock_seq_hi_and_reserved = (uint8_t)((gClockSeq & 0x3F00) >> 8); + // Вариант + uuid->clock_seq_hi_and_reserved |= 0x80; + + // Узел + memmove(&uuid->node[0], &gNode[0], 6); + + return true; +} diff --git a/gpcontrib/uuid_cb/src/uuid.h b/gpcontrib/uuid_cb/src/uuid.h new file mode 100644 index 00000000000..84e95034e8a --- /dev/null +++ b/gpcontrib/uuid_cb/src/uuid.h @@ -0,0 +1,48 @@ +/* The original file is taken from https://www.cbr.ru/ckki/assignment_unique_id/ */ + +/* =================================================================== * + * Вариант реализации функции генерации первой части уникального * + * идентификатора договора (сделки) - универсального уникального * + * идентификатора в соответствии с указанием Банка России * + * "О правилах присвоения уникального идентификатора договора (сделки),* + * по обязательствам из которого (из которой) формируется кредитная * + * история" * + * =================================================================== */ + +#include +#include + +#ifdef uuid_t +#undef uuid_t +#endif + +typedef struct +{ + uint32_t time_low; + uint16_t time_mid; + uint16_t time_hi_and_version; + uint8_t clock_seq_hi_and_reserved; + uint8_t clock_seq_low; + uint8_t node[6]; +} uuid_t; + +// Инициализация модуля генерации УУИд +// Данную функцию необходимо вызвать один раз перед первой генерацией УУИд +// Возвращаемый результат: +// - true - инициализация выполнена успешно +// - false - не удалось выполнить инициализацию +// Если инициализация не выполнена, функция uuid_create() будет завершаться +// ошибкой или возвращать некорректный результат +bool uuid_init(void); + +// Деинициализация модуля генерации УУИд (освобождение ресурсов) +// Данную функция необходимо вызвать перед завершением работы приложения +// для освобождения ресурсов, занятых модулем. +void uuid_deinit(void); + +// Генерация УУИд +// uuid - указатель на структуру uuid_t для помещения в нее данные УУИд +// Возвращаемый результат: +// - true - если УУИд создан успешно и помещен в структуру +// - false - если УУИд создать не удалось +bool uuid_create(uuid_t *uuid); diff --git a/gpcontrib/uuid_cb/test/expected/uuid_cb.out b/gpcontrib/uuid_cb/test/expected/uuid_cb.out new file mode 100644 index 00000000000..73ed5d3ba6d --- /dev/null +++ b/gpcontrib/uuid_cb/test/expected/uuid_cb.out @@ -0,0 +1,63 @@ +CREATE EXTENSION "uuid-cb"; +-- works with NULL +SELECT uuid_cb_valid(NULL); + uuid_cb_valid +--------------- + +(1 row) + +-- valid UUIDs +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-1'); + uuid_cb_valid +--------------- + t +(1 row) + +SELECT uuid_cb_valid('01000000-0000-0000-0000-000000000000-2'); + uuid_cb_valid +--------------- + t +(1 row) + +-- invalid UUIDs +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-2'); + uuid_cb_valid +--------------- + f +(1 row) + +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-1 '); + uuid_cb_valid +--------------- + f +(1 row) + +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000'); + uuid_cb_valid +--------------- + f +(1 row) + +SELECT uuid_cb_valid('foobar'); + uuid_cb_valid +--------------- + f +(1 row) + +-- check uniqueness of generated UUIDs +SELECT COUNT(DISTINCT uid) = 100000 AS no_duplicates FROM + (SELECT uuid_cb_generate() FROM generate_series(1, 100000)) as uid; + no_duplicates +--------------- + t +(1 row) + +-- check correctness of generated UUIDs +SELECT COUNT(1) = 0 AS all_correct FROM + (SELECT uuid_cb_valid(uuid_cb_generate()) as is_correct FROM generate_series(1, 100000)) as subq + where subq.is_correct = False; + all_correct +------------- + t +(1 row) + diff --git a/gpcontrib/uuid_cb/test/sql/uuid_cb.sql b/gpcontrib/uuid_cb/test/sql/uuid_cb.sql new file mode 100644 index 00000000000..5e53e2f2355 --- /dev/null +++ b/gpcontrib/uuid_cb/test/sql/uuid_cb.sql @@ -0,0 +1,23 @@ +CREATE EXTENSION "uuid-cb"; + +-- works with NULL +SELECT uuid_cb_valid(NULL); + +-- valid UUIDs +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-1'); +SELECT uuid_cb_valid('01000000-0000-0000-0000-000000000000-2'); + +-- invalid UUIDs +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-2'); +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000-1 '); +SELECT uuid_cb_valid('10000000-0000-0000-0000-000000000000'); +SELECT uuid_cb_valid('foobar'); + +-- check uniqueness of generated UUIDs +SELECT COUNT(DISTINCT uid) = 100000 AS no_duplicates FROM + (SELECT uuid_cb_generate() FROM generate_series(1, 100000)) as uid; + +-- check correctness of generated UUIDs +SELECT COUNT(1) = 0 AS all_correct FROM + (SELECT uuid_cb_valid(uuid_cb_generate()) as is_correct FROM generate_series(1, 100000)) as subq + where subq.is_correct = False; diff --git a/gpcontrib/uuid_cb/uuid-cb.control b/gpcontrib/uuid_cb/uuid-cb.control new file mode 100644 index 00000000000..cb9849fccf0 --- /dev/null +++ b/gpcontrib/uuid_cb/uuid-cb.control @@ -0,0 +1,6 @@ +# uuid-cb extension +comment = 'generate universally unique identifiers that follow Russian Central Bank requirements' +default_version = '1.0' +module_pathname = '$libdir/uuid-cb' +relocatable = true +trusted = true diff --git a/pom.xml b/pom.xml index 51a7830d5ae..cd19fcd018a 100644 --- a/pom.xml +++ b/pom.xml @@ -158,6 +158,13 @@ code or new licensing patterns. gpcontrib/gp_url_tools/Makefile gpcontrib/gp_url_tools/gp_url_tools.control + gpcontrib/uuid_cb/src/uid.c + gpcontrib/uuid_cb/src/uid.h + gpcontrib/uuid_cb/src/uuid.c + gpcontrib/uuid_cb/src/uuid.h + gpcontrib/uuid_cb/Makefile + gpcontrib/uuid_cb/uuid-cb.control + getversion .git-blame-ignore-revs .dir-locals.el From a65b1168ff62190187fbb6a9a33e8b9a53c74b2b Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:04:00 +0300 Subject: [PATCH 162/167] Import gp_relaccess_stats into gpcontrib from Greenplum (#50) * Import gp_relaccess_stats into gpcontrib from Greenplum * Feat: Adapt gp_relaccess_stats for Cloudberry * Fix: Switch to the old API --------- Co-authored-by: Maxim Smyatkin --- .github/workflows/build-cloudberry-rocky8.yml | 19 + .github/workflows/build-cloudberry.yml | 19 + gpcontrib/Makefile | 2 + gpcontrib/gp_relaccess_stats/.gitignore | 5 + gpcontrib/gp_relaccess_stats/Makefile | 19 + gpcontrib/gp_relaccess_stats/README.md | 105 +++ .../gp_relaccess_stats.control | 6 + .../sql/gp_relaccess_stats--1.0--1.1.sql | 52 ++ .../sql/gp_relaccess_stats--1.1.sql | 143 +++ .../src/gp_relaccess_stats.c | 818 ++++++++++++++++++ .../test/expected/gp_relaccess_stats.out | 406 +++++++++ .../test/sql/gp_relaccess_stats.sql | 186 ++++ pom.xml | 6 +- 13 files changed, 1785 insertions(+), 1 deletion(-) mode change 100644 => 100755 gpcontrib/Makefile create mode 100644 gpcontrib/gp_relaccess_stats/.gitignore create mode 100755 gpcontrib/gp_relaccess_stats/Makefile create mode 100644 gpcontrib/gp_relaccess_stats/README.md create mode 100644 gpcontrib/gp_relaccess_stats/gp_relaccess_stats.control create mode 100644 gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.0--1.1.sql create mode 100755 gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.1.sql create mode 100755 gpcontrib/gp_relaccess_stats/src/gp_relaccess_stats.c create mode 100644 gpcontrib/gp_relaccess_stats/test/expected/gp_relaccess_stats.out create mode 100644 gpcontrib/gp_relaccess_stats/test/sql/gp_relaccess_stats.sql diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index bc0c8146a6d..fe73ca98364 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -324,6 +324,11 @@ jobs: "gpcontrib/gp_url_tools:installcheck", "gpcontrib/uuid_cb:installcheck"] }, + {"test":"gpcontrib-gp-relaccess-stats", + "make_configs":["gpcontrib/gp_relaccess_stats:installcheck"], + "extension":"gp_relaccess_stats", + "shared_preload_libraries":"gp_relaccess_stats" + }, {"test":"gpcontrib-gp-stats-collector", "make_configs":["gpcontrib/gp_stats_collector:installcheck"], "extension":"gp_stats_collector" @@ -1449,6 +1454,20 @@ jobs: exit 1 fi ;; + gp_relaccess_stats) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_relaccess_stats' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_relaccess_stats; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_relaccess_stats extension" + exit 1 + fi + ;; *) echo "Unknown extension: ${{ matrix.extension }}" exit 1 diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index c20481da7b1..419aa4365a6 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -271,6 +271,11 @@ jobs: }, "enable_core_check":false }, + {"test":"gpcontrib-gp-relaccess-stats", + "make_configs":["gpcontrib/gp_relaccess_stats:installcheck"], + "extension":"gp_relaccess_stats", + "shared_preload_libraries":"gp_relaccess_stats" + }, {"test":"gpcontrib-gp-stats-collector", "make_configs":["gpcontrib/gp_stats_collector:installcheck"], "extension":"gp_stats_collector" @@ -1462,6 +1467,20 @@ jobs: exit 1 fi ;; + gp_relaccess_stats) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_relaccess_stats' && \ + gpstop -ra && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_relaccess_stats; \ + SHOW shared_preload_libraries; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_relaccess_stats extension" + exit 1 + fi + ;; *) echo "Unknown extension: ${{ matrix.extension }}" exit 1 diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile old mode 100644 new mode 100755 index fa1115a1cb5..43844a5d0f5 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -20,6 +20,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_inject_fault \ gp_exttable_fdw \ gp_legacy_string_agg \ + gp_relaccess_stats \ gp_replica_check \ gp_toolkit \ gp_url_tools \ @@ -30,6 +31,7 @@ else gp_distribution_policy \ gp_internal_tools \ gp_legacy_string_agg \ + gp_relaccess_stats \ gp_exttable_fdw \ gp_toolkit \ gp_url_tools \ diff --git a/gpcontrib/gp_relaccess_stats/.gitignore b/gpcontrib/gp_relaccess_stats/.gitignore new file mode 100644 index 00000000000..8031bbbf6eb --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/.gitignore @@ -0,0 +1,5 @@ +*.o +*.so +.vscode +compile_commands.json +results diff --git a/gpcontrib/gp_relaccess_stats/Makefile b/gpcontrib/gp_relaccess_stats/Makefile new file mode 100755 index 00000000000..eb973c33c34 --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/Makefile @@ -0,0 +1,19 @@ +MODULE_big = gp_relaccess_stats +OBJS = ./src/gp_relaccess_stats.o +EXTENSION = gp_relaccess_stats +EXTVERSION = 1.0 +DATA = $(wildcard sql/*--*.sql) +REGRESS = gp_relaccess_stats +REGRESS_OPTS = --inputdir=test/ +PGFILEDESC = "gp_relaccess_stats - facility to track how and when tables, partitions or views were accessed" +PG_CXXFLAGS += $(COMMON_CPP_FLAGS) + +ifdef USE_PGXS + PG_CONFIG = pg_config + PGXS := $(shell $(PG_CONFIG) --pgxs) + include $(PGXS) +else + top_builddir = ../.. + include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/gpcontrib/gp_relaccess_stats/README.md b/gpcontrib/gp_relaccess_stats/README.md new file mode 100644 index 00000000000..4d191f8fef2 --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/README.md @@ -0,0 +1,105 @@ + + +# gp_relaccess_stats: Table access monitoring tool for Greenplum + +## Features +gp_relaccess_stats is an extension that records access statistics for Greenplum tables and views. Allowing users to see what objects were used, when and by whom. For example, this allows DBAs to find objects that are not used anymore or objects that are being misused. + +Features include: +* support of both tables (regular, external or partitioned) and views +* separate tracking of select, insert, update and delete queries +* separate tracking of last read and write timestamps +* tracking of the last user who accessed the object +* per-database configuration +* in-memory stats survive server restarts (but not crashes) + +### Supported versions and platforms +For now it is being tested only for GP6 and Linux. Though, there are no apparent reasons why it should not be working on newer GP versions (or even PG with slight code modification) or other OSes. + +### Installation +Install from source: +```bash +# get the source code somewhere +git clone git@github.com:Smyatkin-Maxim/gp_relaccess_stats.git +cd gp_relaccess_stats +# Build it. Building would require GP installed nearby and sourcing greenplum_path.sh +source /greenplum_path.sh +make && make install +``` + +### Configuration +As this extension does extensive usage of hooks and shared memory, you need to load gp_relaccess_stats.so on start-up: +``` +gpconfig -c shared_preload_libraries -v 'gp_relaccess_stats' && gpstop -ra +``` +gp_relaccess_stats configuration parameters: +| **Parameter** | **Type** | **Default** | **Default** | +| ---------------- | --------------- | ------------ | ------------ | +| `gp_relaccess_stats.enabled` | bool | false | Using `gp_relaccess_stats.enabled` you can enable/disable stats collection either globally or for each database separately. The second option is preferred.| +| `gp_relaccess_stats.max_tables` | integer | 65536 | `gp_relaccess_stats.max_tables` is a hard limit on how many tables can be cached in shared memory. Feel free to make this number higher if necessary, as the overhead is only about 160 bytes per table. Note, that stats cache for a specific table is evicted from memory any time you execute `relaccess_stats_update()` or `relaccess_stats_dump()` and new tables can be recorded. If you call these functions often enough, there is no need for high gp_relaccess_stats.max_tables| +| `gp_relaccess_stats.dump_on_overflow` | bool | false | This parameter configures what happens in case `gp_relaccess_stats.max_tables` was not enough. If set to `true`, `relaccess_stats_dump()` will be called implicitly and stats cache will be freed. Otherwice, you will get a WARNING saying that there is no room for new stats. Is this case, stats for some tables will be lost.| + +### Usage +The first thing you need to do after `CREATE EXTENSION` and configuring - execute `SELECT relaccess_stats_init();` in a specific database. This function will fill `relaccess_stats` table with empty stats for each table and partition in this database. This is optional, but will come handy when you try to find tables that haven't been used recently, for example. + +Then, either manually or with a cron job start executing `select relaccess_stats_update()`. This function takes all stats cached in shared memory and all stats stored in pg_stat dir (e.g, dumps after restarts, or when `max_tables` was exceeded) and upserts them into `relaccess_stats` table. + +The `relaccess_stats` table itself looks like this: +| **Column** | **Description** | +| ---------------- | --------------- | +| relid | OID of the relation | +| relname | Name of the relation at last access | +| last_reader_id | OID of user who read the table last | +| last_writer_id | OID of user who wrote the table last | +| last_read | Timestamp of the most recent select | +| last_write | Timestamp of the most recent insert/delete/update/truncate | +| n_select_queries | | +| n_insert_queries | | +| n_update_queries | | +| n_delete_queries | | +| n_truncate_queries | | + +**NOTE**: n_*_queries columns count the number of queries executed, not the number of rows read, inserted, deleted or updated. + +This table has a view associated with it: `relaccess_stats_root_tables_aggregated`. This view has exactly same columns, however it only shows partitioned tables. To be more specific, it shows aggregated stats for each partitioned table. +For example, if we have 1 insert into `tbl1_prt_1` and 3 inserts into `tbl1_prt_2`, then `select * from relaccess_stats_root_tables_aggregated where relname = 'tbl1'` will show us only root table with n_insert_queries = 4. This view, however, has some limitations. See the next section for more detail. + +Another useful function is `relaccess_stats_dump()`, which simply moves cached stats from shared memory to temporary files in pg_stat directory. This function is cheaper than `relaccess_stats_update` but will evict stats cache if needed. Though, stats in temporary files can also get lost. Hence, it is recommended to stick with frequent `select relaccess_stats_update()` calls. + +To better understand when it's time to dump or update the stats one might check `select relaccess.relaccess_stats_fillfactor();`. It will show current usage of stats hash table in percents. For example if shared memory for our relaccess hash table is 70% full we will get relaccess_stats_fillfactor=70. It would be a good idea to dump or update when fillfactor is around 70%. + +### Limitations and gotchas +There is a number of interesting edge-cases in this simple extension: +* `relaccess_stats_root_tables_aggregated` shows info only about tables that exist **now**. We simply can`t get information about inheritance relationship for deleted tables. +* Stats don't rollback on savepoint rollback. We will see n_select_queries incremented by 2 in the following case: +```sql +BEGIN; +SELECT * FROM tbl; +SAVEPOINT sp; +SELECT * FROM tbl; +ROLLBACK TO SAVEPOINT sp; +COMMIT; +``` +There is no technical reason for this limitation. It cat be fixed when there will be need for that. +* Update stats often! Otherwise, data can be lost if any of it happens: 1) there was a crash, 2) `max_tables` exceeded w/o `dump_on_overflow`, 3) temporary pg_stat dir got cleaned. +* no `truncate only` support. There is a TODO in code in case it is ever needed. +* Updates and Deletes also increment n_select_queries. Every update and delete also read the table. That is, n_select_queries get incremented as well. If you need **only** selects, query like this `SELECT n_select_queries - (n_update_queries + n_delete_queries) ... FROM relaccess_stats ...;`. For this same reason last_read and last_reader_id change on update and delete queries. +* view = view + tables. It looks like whenever you select from view, n_select_queries get incremented for both the view and tables it references. +* obviously, we don't know any timestamps before we started tracking. So, the first timestamps are initialized with 0 (something around year 2000), which means those tables haven't been accessed since gp_relaccess_stats was enabled. diff --git a/gpcontrib/gp_relaccess_stats/gp_relaccess_stats.control b/gpcontrib/gp_relaccess_stats/gp_relaccess_stats.control new file mode 100644 index 00000000000..e1c5f5189b9 --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/gp_relaccess_stats.control @@ -0,0 +1,6 @@ +# gp_relaccess_stats extension +comment = 'gp_relaccess_stats - facility to track how and when tables, partitions or views were accesseds' +default_version = '1.1' +module_pathname = '$libdir/gp_relaccess_stats' +relocatable = true +trusted = true diff --git a/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.0--1.1.sql b/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.0--1.1.sql new file mode 100644 index 00000000000..799474e6f2f --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.0--1.1.sql @@ -0,0 +1,52 @@ +/* gp_relaccess_stats--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION gp_relaccess_stats UPDATE TO '1.1'" to load this file. \quit + +DROP VIEW relaccess.relaccess_stats_root_tables_aggregated; + +ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_select_queries TYPE int8; +ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_insert_queries TYPE int8; +ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_update_queries TYPE int8; +ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_delete_queries TYPE int8; +ALTER TABLE relaccess.relaccess_stats ALTER COLUMN n_truncate_queries TYPE int8; + +-- This utility view shows **ONLY** stats on **EXISTING** partitioned tables in aggregated form +CREATE VIEW relaccess.relaccess_stats_root_tables_aggregated AS ( + WITH RECURSIVE parents AS ( + SELECT inhrelid AS child, inhparent AS parent FROM pg_inherits + UNION ALL + SELECT prev.child, next.inhparent AS parent FROM parents AS prev JOIN pg_inherits AS next ON prev.parent = next.inhrelid + ), part_to_root_mapping AS ( + SELECT DISTINCT child AS partid, min(parent) OVER (partition BY child) AS rootid FROM parents + ), parts_including_roots AS ( + SELECT rootid as partid, rootid FROM (SELECT DISTINCT rootid FROM part_to_root_mapping) AS p + UNION + SELECT * FROM part_to_root_mapping + ), with_root_id AS ( + SELECT part_tbl.rootid, stats.* FROM relaccess.relaccess_stats stats JOIN parts_including_roots part_tbl ON (stats.relid = part_tbl.partid) + ), without_last_user AS ( + SELECT rootid AS relid, + rootid::regclass::text AS relname, + max(last_read) AS last_read, + max(last_write) AS last_write, + sum(n_select_queries) AS n_select_queries, + sum(n_insert_queries) AS n_insert_queries, + sum(n_update_queries) AS n_update_queries, + sum(n_delete_queries) AS n_delete_queries, + sum(n_truncate_queries) AS n_truncate_queries + FROM with_root_id outer_tbl GROUP BY rootid + ) + SELECT relid, + relname, + (SELECT last_reader_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id, + (SELECT last_writer_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id, + last_read, + last_write, + n_select_queries, + n_insert_queries, + n_update_queries, + n_delete_queries, + n_truncate_queries + FROM without_last_user wo +); diff --git a/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.1.sql b/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.1.sql new file mode 100755 index 00000000000..6d25b63132d --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/sql/gp_relaccess_stats--1.1.sql @@ -0,0 +1,143 @@ +/* gp_relaccess_stats--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_relaccess_stats" to load this file. \quit + +CREATE SCHEMA IF NOT EXISTS relaccess; + +CREATE TABLE relaccess.relaccess_stats ( + relid Oid, + relname Name, + last_reader_id Oid, + last_writer_id Oid, + last_read timestamptz, + last_write timestamptz, + n_select_queries int8, + n_insert_queries int8, + n_update_queries int8, + n_delete_queries int8, + n_truncate_queries int8 +) DISTRIBUTED BY (relid); + +CREATE FUNCTION relaccess.relaccess_stats_dump() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'relaccess_stats_dump' +LANGUAGE C VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION relaccess.relaccess_stats_update() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'relaccess_stats_update' +LANGUAGE C VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION relaccess.relaccess_stats_fillfactor() +RETURNS SETOF INT2 +AS 'MODULE_PATHNAME', 'relaccess_stats_fillfactor' +LANGUAGE C VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION relaccess.__get_db_stats_from_dump() +RETURNS SETOF relaccess.relaccess_stats +AS 'MODULE_PATHNAME', 'relaccess_stats_from_dump' +LANGUAGE C VOLATILE EXECUTE ON MASTER; + +CREATE FUNCTION relaccess.__relaccess_upsert_from_dump_file() RETURNS VOID +LANGUAGE plpgsql VOLATILE AS +$func$ +BEGIN + EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp'; + EXECUTE 'CREATE TEMP TABLE relaccess_stats_tmp (LIKE relaccess.relaccess_stats) distributed by (relid)'; + EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp_aggregated'; + EXECUTE 'CREATE TEMP TABLE relaccess_stats_tmp_aggregated (LIKE relaccess.relaccess_stats) distributed by (relid)'; + EXECUTE 'INSERT INTO relaccess_stats_tmp SELECT * FROM relaccess.__get_db_stats_from_dump()'; + EXECUTE 'WITH aggregated_wo_relname_and_user AS ( + SELECT relid, max(last_read) AS last_read, max(last_write) AS last_write, sum(n_select_queries) AS n_select_queries, + sum(n_insert_queries) AS n_insert_queries, sum(n_update_queries) AS n_update_queries, sum(n_delete_queries) AS n_delete_queries, sum(n_truncate_queries) AS n_truncate_queries + FROM relaccess_stats_tmp GROUP BY relid + ) + INSERT INTO relaccess_stats_tmp_aggregated + SELECT relid, + (SELECT relname FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND greatest(wo.last_read, wo.last_write) IN (w.last_read, w.last_write) LIMIT 1) AS relname, + (SELECT last_reader_id FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id, + (SELECT last_writer_id FROM relaccess_stats_tmp w WHERE w.relid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id, + last_read, + last_write, + n_select_queries, + n_insert_queries, + n_update_queries, + n_delete_queries, + n_truncate_queries FROM aggregated_wo_relname_and_user AS wo'; + EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp'; + EXECUTE 'INSERT INTO relaccess.relaccess_stats + SELECT relid, relname, last_reader_id, last_writer_id, last_read, last_write, 0, 0, 0, 0, 0 + FROM relaccess_stats_tmp_aggregated stage + WHERE NOT EXISTS ( + SELECT 1 FROM relaccess.relaccess_stats orig WHERE orig.relid = stage.relid)'; + EXECUTE 'UPDATE relaccess.relaccess_stats orig SET + relname = stage.relname, + n_select_queries = orig.n_select_queries + stage.n_select_queries, + n_insert_queries = orig.n_insert_queries + stage.n_insert_queries, + n_update_queries = orig.n_update_queries + stage.n_update_queries, + n_delete_queries = orig.n_delete_queries + stage.n_delete_queries, + n_truncate_queries = orig.n_truncate_queries + stage.n_truncate_queries + FROM relaccess_stats_tmp_aggregated stage + WHERE orig.relid = stage.relid'; + EXECUTE 'UPDATE relaccess.relaccess_stats orig SET + last_reader_id = stage.last_reader_id, last_read = stage.last_read + FROM relaccess_stats_tmp_aggregated stage + WHERE orig.relid = stage.relid AND orig.last_read < stage.last_read'; + EXECUTE 'UPDATE relaccess.relaccess_stats orig SET + last_writer_id = stage.last_writer_id, last_write = stage.last_write + FROM relaccess_stats_tmp_aggregated stage + WHERE orig.relid = stage.relid AND orig.last_write < stage.last_write'; + EXECUTE 'DROP TABLE IF EXISTS relaccess_stats_tmp_aggregated'; +END +$func$; + +CREATE FUNCTION relaccess.relaccess_stats_init() RETURNS VOID AS +$$ + WITH relations AS ( + SELECT oid as relid, relname, relowner FROM pg_catalog.pg_class WHERE relkind in ('r', 'v', 'm', 'f', 'p') + ) + INSERT INTO relaccess.relaccess_stats + SELECT relid, relname, relowner, relowner, '2000-01-01 03:00:00', '2000-01-01 03:00:00', 0, 0, 0, 0, 0 + FROM relations AS all_rels WHERE NOT EXISTS(SELECT 1 FROM relaccess.relaccess_stats orig WHERE orig.relid = all_rels.relid); +$$ LANGUAGE SQL VOLATILE; + +-- This utility view shows **ONLY** stats on **EXISTING** partitioned tables in aggregated form +CREATE VIEW relaccess.relaccess_stats_root_tables_aggregated AS ( + WITH RECURSIVE parents AS ( + SELECT inhrelid AS child, inhparent AS parent FROM pg_inherits + UNION ALL + SELECT prev.child, next.inhparent AS parent FROM parents AS prev JOIN pg_inherits AS next ON prev.parent = next.inhrelid + ), part_to_root_mapping AS ( + SELECT DISTINCT child AS partid, min(parent) OVER (partition BY child) AS rootid FROM parents + ), parts_including_roots AS ( + SELECT rootid as partid, rootid FROM (SELECT DISTINCT rootid FROM part_to_root_mapping) AS p + UNION + SELECT * FROM part_to_root_mapping + ), with_root_id AS ( + SELECT part_tbl.rootid, stats.* FROM relaccess.relaccess_stats stats JOIN parts_including_roots part_tbl ON (stats.relid = part_tbl.partid) + ), without_last_user AS ( + SELECT rootid AS relid, + rootid::regclass::text AS relname, + max(last_read) AS last_read, + max(last_write) AS last_write, + sum(n_select_queries) AS n_select_queries, + sum(n_insert_queries) AS n_insert_queries, + sum(n_update_queries) AS n_update_queries, + sum(n_delete_queries) AS n_delete_queries, + sum(n_truncate_queries) AS n_truncate_queries + FROM with_root_id outer_tbl GROUP BY rootid + ) + SELECT relid, + relname, + (SELECT last_reader_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_read = w.last_read LIMIT 1) AS last_reader_id, + (SELECT last_writer_id FROM with_root_id w WHERE w.rootid = wo.relid AND wo.last_write = w.last_write LIMIT 1) AS last_writer_id, + last_read, + last_write, + n_select_queries, + n_insert_queries, + n_update_queries, + n_delete_queries, + n_truncate_queries + FROM without_last_user wo +); diff --git a/gpcontrib/gp_relaccess_stats/src/gp_relaccess_stats.c b/gpcontrib/gp_relaccess_stats/src/gp_relaccess_stats.c new file mode 100755 index 00000000000..031b1d830c3 --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/src/gp_relaccess_stats.c @@ -0,0 +1,818 @@ +#include "postgres.h" +#include "access/table.h" +#include "access/xact.h" +#include "access/hash.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_database.h" +#include "cdb/cdbvars.h" +#include "commands/dbcommands.h" +#include "executor/executor.h" +#include "executor/spi.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pg_config_ext.h" +#include "pgstat.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/builtins.h" +#include "utils/datetime.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/timestamp.h" +#include "tcop/utility.h" + +#include +#include +#include + +/** + * gp_relaccess_stats collects runtime access stats on db objects: relations and + * views. Stats include last read and write timestamps, last user, last known + * relname and number of select, insert, update, delete or truncate queries. + * Only committed actions are recorded. + * + * To track those actions we use: + * - ExecutorCheckPerms hook for select, insert, update and delete statements + * - ProcessUtility hook for truncate statements + * + * Intermediate data is stored in three hash tables. + * One lives in shared memory and is cleaned only when dumped to disc: + * - relaccesses - represents all recorded accesses since last dump to disc. + * And two live in coordinator`s local memory and are cleaned on every commit + * or rollback: + * - local_access_entries - represent all record accesses in for this + * transaction only + * - relname_cache - maps relid to relname for relations used in this + * transaction only + * + * Ultimately all recorded stats should end up in relaccess_stats table when a + * user executes relaccess_stats_update(). But any intermediate stats will be + * dumped to disc. This might happen for either or those reasons: + * - shmem is exceeded + * - server is restarted + * - manual execution of relaccess_stats_dump() + * In this case stats are offloaded to disc into pg_stat directory into separate + * file per each tracked database: pg_stat/relaccess_stats_dump_.csv Those + * files are upserted into relaccess_stats when relaccess_stats_update() is + * called + */ + +PG_MODULE_MAGIC; + +void _PG_init(void); +void _PG_fini(void); +PG_FUNCTION_INFO_V1(relaccess_stats_update); +PG_FUNCTION_INFO_V1(relaccess_stats_dump); +PG_FUNCTION_INFO_V1(relaccess_stats_fillfactor); +PG_FUNCTION_INFO_V1(relaccess_stats_from_dump); + +static void relaccess_stats_update_internal(void); +static void relaccess_dump_to_files(bool only_this_db); +static void relaccess_dump_to_files_internal(HTAB *files); +static void relaccess_upsert_from_file(void); +static void relaccess_shmem_startup(void); +static void relaccess_shmem_shutdown(int code, Datum arg); +static uint32 relaccess_hash_fn(const void *key, Size keysize); +static int relaccess_match_fn(const void *key1, const void *key2, Size keysize); +static uint32 local_relaccess_hash_fn(const void *key, Size keysize); +static int local_relaccess_match_fn(const void *key1, const void *key2, + Size keysize); +static bool collect_relaccess_hook(List *rangeTable, + bool ereport_on_violation); +static void relaccess_xact_callback(XactEvent event, void *arg); +static void collect_truncate_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc); +static void relaccess_executor_end_hook(QueryDesc *query_desc); +static void relaccess_drop_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg); +static void memorize_local_access_entry(Oid relid, AclMode perms); +static void update_relname_cache(Oid relid, char *relname); +static StringInfoData get_dump_filename(Oid dbid); + +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; +static ExecutorCheckPerms_hook_type prev_check_perms_hook = NULL; +static ProcessUtility_hook_type next_ProcessUtility_hook = NULL; +static ExecutorEnd_hook_type prev_ExecutorEnd_hook = NULL; +static object_access_hook_type prev_object_access_hook = NULL; + +typedef struct relaccessHashKey { + Oid dbid; + Oid relid; +} relaccessHashKey; + +typedef struct relaccessEntry { + relaccessHashKey key; + char relname[NAMEDATALEN]; + Oid last_reader_id; + Oid last_writer_id; + TimestampTz last_read; + TimestampTz last_write; + int64 n_select; + int64 n_insert; + int64 n_update; + int64 n_delete; + int64 n_truncate; +} relaccessEntry; + +typedef struct relaccessGlobalData { + LWLock *relaccess_ht_lock; + LWLock *relaccess_file_lock; +} relaccessGlobalData; + +typedef struct localAccessKey { + Oid relid; + int stmt_cnt; +} localAccessKey; + +typedef struct localAccessEntry { + localAccessKey key; + Oid last_reader_id, last_writer_id; + Timestamp last_read, last_write; + AclMode perms; +} localAccessEntry; + +typedef struct relnameCacheEntry { + Oid relid; + char relname[NAMEDATALEN]; +} relnameCacheEntry; + +typedef struct fileDumpEntry { + Oid dbid; + char *filename; + FILE *file; +} fileDumpEntry; + +static int32 relaccess_size; +static bool dump_on_overflow; +static bool is_enabled; +static relaccessGlobalData *data; +static HTAB *relaccesses; +static HTAB *local_access_entries = NULL; +static const int32 LOCAL_HTAB_SZ = 128; +static HTAB *relname_cache = NULL; +static const int32 RELCACHE_SZ = 16; +static const int32 FILE_CACHE_SZ = 16; +static int stmt_counter = 0; +static bool had_ht_overflow = false; + +#define IS_POSTGRES_DB \ + (strcmp("postgres", get_database_name(MyDatabaseId)) == 0) + +#define is_write(perms) \ + (((perms) & (ACL_INSERT | ACL_UPDATE | ACL_DELETE | ACL_TRUNCATE)) != 0) + +#define is_read(perms) (!is_write(perms) && ((perms) & ACL_SELECT) != 0) + +static void relaccess_shmem_startup() { + bool found; + HASHCTL info; + + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + data = (relaccessGlobalData *)(ShmemInitStruct( + "relaccess_stats", sizeof(relaccessGlobalData), &found)); + if (!found) { + LWLockPadded *locks = GetNamedLWLockTranche("gp_relaccess_stats"); + data->relaccess_ht_lock = &locks[0].lock; + data->relaccess_file_lock = &locks[1].lock; + } + + memset(&info, 0, sizeof(info)); + info.keysize = sizeof(relaccessHashKey); + info.entrysize = sizeof(relaccessEntry); + info.hash = relaccess_hash_fn; + info.match = relaccess_match_fn; + relaccesses = ShmemInitHash( + "relaccess_stats hash", relaccess_size, relaccess_size, &info, + (HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_FIXED_SIZE)); + + LWLockRelease(AddinShmemInitLock); + + if (!IsUnderPostmaster) { + on_shmem_exit(relaccess_shmem_shutdown, (Datum)0); + } +} + +static void relaccess_shmem_shutdown(int code, Datum arg) { + if (code || !data || !relaccesses) { + return; + } + LWLockAcquire(data->relaccess_ht_lock, LW_EXCLUSIVE); + relaccess_dump_to_files(false); + LWLockRelease(data->relaccess_ht_lock); +} + +static uint32 relaccess_hash_fn(const void *key, Size keysize) { + const relaccessHashKey *k = (const relaccessHashKey *)key; + return hash_uint32((uint32)k->dbid) ^ hash_uint32((uint32)k->relid); +} + +static int relaccess_match_fn(const void *key1, const void *key2, + Size keysize) { + const relaccessHashKey *k1 = (const relaccessHashKey *)key1; + const relaccessHashKey *k2 = (const relaccessHashKey *)key2; + return (k1->dbid == k2->dbid && k1->relid == k2->relid) ? 0 : 1; +} + +static uint32 local_relaccess_hash_fn(const void *key, Size keysize) { + const localAccessKey *k = (const localAccessKey *)key; + return hash_uint32((uint32)k->stmt_cnt) ^ hash_uint32((uint32)k->relid); +} + +static int local_relaccess_match_fn(const void *key1, const void *key2, + Size keysize) { + const localAccessKey *k1 = (const localAccessKey *)key1; + const localAccessKey *k2 = (const localAccessKey *)key2; + return (k1->stmt_cnt == k2->stmt_cnt && k1->relid == k2->relid) ? 0 : 1; +} + +void _PG_init(void) { + if (!process_shared_preload_libraries_in_progress) { + return; + } + + DefineCustomIntVariable( + "gp_relaccess_stats.max_tables", + "Sets the maximum number of tables cached by gp_relaccess_stats.", NULL, + &relaccess_size, 65536, 128, INT_MAX, PGC_POSTMASTER, 0, NULL, NULL, + NULL); + + DefineCustomBoolVariable("gp_relaccess_stats.dump_on_overflow", + "Selects whether we should dump to .csv in case " + "gp_relaccess_stats.max_tables is exceeded.", + NULL, &dump_on_overflow, false, PGC_SIGHUP, 0, NULL, + NULL, NULL); + + DefineCustomBoolVariable( + "gp_relaccess_stats.enabled", + "Collect table access stats globally or for a specific database. " + "Note that shared memory is initialized indepemdent of this argument.", + NULL, &is_enabled, false, PGC_SUSET, 0, NULL, NULL, NULL); + + if (Gp_role != GP_ROLE_DISPATCH) { + return; + } + + RequestNamedLWLockTranche("gp_relaccess_stats", 2); + Size size = MAXALIGN(sizeof(relaccessGlobalData)); + size = add_size(size, + hash_estimate_size(relaccess_size, sizeof(relaccessEntry))); + RequestAddinShmemSpace(size); + + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = relaccess_shmem_startup; + prev_check_perms_hook = ExecutorCheckPerms_hook; + ExecutorCheckPerms_hook = collect_relaccess_hook; + next_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = collect_truncate_hook; + prev_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = relaccess_executor_end_hook; + prev_object_access_hook = object_access_hook; + object_access_hook = relaccess_drop_hook; + RegisterXactCallback(relaccess_xact_callback, NULL); + HASHCTL ctl; + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(localAccessKey); + ctl.entrysize = sizeof(localAccessEntry); + ctl.hash = local_relaccess_hash_fn; + ctl.match = local_relaccess_match_fn; + local_access_entries = + hash_create("Transaction-wide relaccess entries", LOCAL_HTAB_SZ, &ctl, + HASH_ELEM | HASH_FUNCTION | HASH_COMPARE); + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(relnameCacheEntry); + ctl.hash = oid_hash; + relname_cache = hash_create("Transaction-wide relation name cache", + RELCACHE_SZ, &ctl, HASH_ELEM | HASH_FUNCTION); +} + +void _PG_fini(void) { + if (Gp_role != GP_ROLE_DISPATCH) { + return; + } + shmem_startup_hook = prev_shmem_startup_hook; + ExecutorCheckPerms_hook = prev_check_perms_hook; + ProcessUtility_hook = next_ProcessUtility_hook; + ExecutorEnd_hook = prev_ExecutorEnd_hook; + object_access_hook = prev_object_access_hook; +} + +static bool collect_relaccess_hook(List *rangeTable, + bool ereport_on_violation) { + if (prev_check_perms_hook && + !prev_check_perms_hook(rangeTable, ereport_on_violation)) { + return false; + } + if (Gp_role == GP_ROLE_DISPATCH && is_enabled) { + ListCell *r; + foreach (r, rangeTable) { + RangeTblEntry *rte = (RangeTblEntry *)lfirst(r); + if (rte->rtekind != RTE_RELATION) { + continue; + } + AclMode requiredPerms = rte->requiredPerms; + if (is_read(requiredPerms) || is_write(requiredPerms)) { + memorize_local_access_entry(rte->relid, requiredPerms); + update_relname_cache(rte->relid, NULL); + } + } + } + return true; +} + +static void collect_truncate_hook(PlannedStmt *pstmt, const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, QueryCompletion *qc) { + Node *parsetree = pstmt->utilityStmt; + if (nodeTag(parsetree) == T_TruncateStmt && is_enabled && + Gp_role == GP_ROLE_DISPATCH) { + TruncateStmt *stmt = (TruncateStmt *)parsetree; + ListCell *cell; + /** + * TODO: TRUNCATE may be called with ONLY option which limits it only to + *the root partition. Otherwise it will truncate all child partitions. We + *might wish to track the difference by explicitly adding records for each + *truncated partition in the future if it proves useful + **/ + foreach (cell, stmt->relations) { + RangeVar *rv = lfirst(cell); + Relation rel = table_openrv(rv, AccessExclusiveLock); + Oid relid = rel->rd_id; + table_close(rel, NoLock); + memorize_local_access_entry(relid, ACL_TRUNCATE); + update_relname_cache(relid, rv->relname); + } + } + if (next_ProcessUtility_hook) { + (*next_ProcessUtility_hook)(pstmt, queryString, readOnlyTree, context, + params, queryEnv, dest, qc); + } else { + standard_ProcessUtility(pstmt, queryString, readOnlyTree, context, params, + queryEnv, dest, qc); + } +} + +#define UPDATE_STAT(lowercase, uppercase) \ + dst_entry->n_##lowercase += (src_entry->perms & ACL_##uppercase ? 1 : 0) + +// if there is a better way to cleanup a postgres hashtable +// w/o recreating it, I didn't find it +#define CLEAR_HTAB(entryType, hmap, key_name) \ + { \ + HASH_SEQ_STATUS hash_seq; \ + entryType *src_entry; \ + hash_seq_init(&hash_seq, hmap); \ + while ((src_entry = hash_seq_search(&hash_seq)) != NULL) { \ + bool found; \ + hash_search(hmap, &src_entry->key_name, HASH_REMOVE, &found); \ + Assert(found); \ + } \ + } + +static void relaccess_xact_callback(XactEvent event, void *arg) { + if (Gp_role != GP_ROLE_DISPATCH || !is_enabled) { + return; + } + // TODO: add support for savepoint rollbacks + Assert(GetCurrentTransactionNestLevel() == 1); + if (event == XACT_EVENT_COMMIT) { + HASH_SEQ_STATUS hash_seq; + localAccessEntry *src_entry; + hash_seq_init(&hash_seq, local_access_entries); + LWLockAcquire(data->relaccess_ht_lock, LW_EXCLUSIVE); + while ((src_entry = hash_seq_search(&hash_seq)) != NULL) { + bool found; + relaccessHashKey key; + key.dbid = MyDatabaseId; + key.relid = src_entry->key.relid; + long n_access_records = hash_get_num_entries(relaccesses); + relaccessEntry *dst_entry = NULL; + Assert(n_access_records <= relaccess_size); + if (n_access_records == relaccess_size) { + // no room for new entries. Perhaps this relid is already being tracked? + dst_entry = + (relaccessEntry *)hash_search(relaccesses, &key, HASH_FIND, &found); + } else { + dst_entry = (relaccessEntry *)hash_search(relaccesses, &key, + HASH_ENTER_NULL, &found); + } + if (dst_entry || dump_on_overflow) { + if (!dst_entry) { + // we are out of shared memory and need to dump + relaccess_dump_to_files(false); + // we MUST have enough space now, unless we were unable to dump + dst_entry = (relaccessEntry *)hash_search(relaccesses, &key, + HASH_ENTER_NULL, &found); + if (!dst_entry) { + // still no memory left + if (!had_ht_overflow) { + elog(WARNING, ("gp_relaccess_stats.max_tables is exceeded and we " + "are unable to dump hashtables to disk. " + "Will start loosing some relaccess stats")); + had_ht_overflow = true; + } + continue; + } else { + had_ht_overflow = false; + } + } + if (!found) { + dst_entry->last_reader_id = InvalidOid; + dst_entry->last_writer_id = InvalidOid; + dst_entry->last_read = 0; + dst_entry->last_write = 0; + dst_entry->n_select = 0; + dst_entry->n_insert = 0; + dst_entry->n_update = 0; + dst_entry->n_delete = 0; + dst_entry->n_truncate = 0; + } + UPDATE_STAT(select, SELECT); + UPDATE_STAT(insert, INSERT); + UPDATE_STAT(update, UPDATE); + UPDATE_STAT(delete, DELETE); + UPDATE_STAT(truncate, TRUNCATE); + if (src_entry->last_read > dst_entry->last_read) { + dst_entry->last_read = src_entry->last_read; + dst_entry->last_reader_id = src_entry->last_reader_id; + } + if (src_entry->last_write > dst_entry->last_write) { + dst_entry->last_write = src_entry->last_write; + dst_entry->last_writer_id = src_entry->last_writer_id; + } + relnameCacheEntry *namecache_entry = (relnameCacheEntry *)hash_search( + relname_cache, &key.relid, HASH_ENTER, &found); + Assert(namecache_entry); + strlcpy(dst_entry->relname, namecache_entry->relname, + sizeof(dst_entry->relname)); + } else { + if (!had_ht_overflow) { + elog(WARNING, "gp_relaccess_stats.max_tables is exceeded! New table " + "events will be lost. " + "Please execute relaccess_stats_update() and consider " + "setting a hihger value"); + } + had_ht_overflow = true; + } + } + LWLockRelease(data->relaccess_ht_lock); + CLEAR_HTAB(localAccessEntry, local_access_entries, key); + CLEAR_HTAB(relnameCacheEntry, relname_cache, relid); + } else if (event == XACT_EVENT_ABORT) { + CLEAR_HTAB(localAccessEntry, local_access_entries, key); + CLEAR_HTAB(relnameCacheEntry, relname_cache, relid); + } +} + +Datum relaccess_stats_update(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + funcctx->max_calls = 1; + relaccess_stats_update_internal(); + } + + funcctx = SRF_PERCALL_SETUP(); + if (funcctx->call_cntr < funcctx->max_calls) { + SRF_RETURN_NEXT(funcctx, (Datum)0); + } + SRF_RETURN_DONE(funcctx); +} + +Datum relaccess_stats_dump(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + funcctx->max_calls = 1; + LWLockAcquire(data->relaccess_ht_lock, LW_EXCLUSIVE); + relaccess_dump_to_files(true); + LWLockRelease(data->relaccess_ht_lock); + } + + funcctx = SRF_PERCALL_SETUP(); + if (funcctx->call_cntr < funcctx->max_calls) { + SRF_RETURN_NEXT(funcctx, (Datum)0); + } + SRF_RETURN_DONE(funcctx); +} + +Datum relaccess_stats_fillfactor(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + + if (SRF_IS_FIRSTCALL()) { + int16_t fillfactor; + + funcctx = SRF_FIRSTCALL_INIT(); + funcctx->max_calls = 1; + LWLockAcquire(data->relaccess_ht_lock, LW_SHARED); + fillfactor = hash_get_num_entries(relaccesses) * 100 / relaccess_size; + LWLockRelease(data->relaccess_ht_lock); + funcctx->user_fctx = (void *)(intptr_t)fillfactor; + } + + funcctx = SRF_PERCALL_SETUP(); + if (funcctx->call_cntr < funcctx->max_calls) { + SRF_RETURN_NEXT(funcctx, + Int16GetDatum((int16_t)(intptr_t)funcctx->user_fctx)); + } + SRF_RETURN_DONE(funcctx); +} + +Datum relaccess_stats_from_dump(PG_FUNCTION_ARGS) { + FuncCallContext *funcctx; + List *stats_entries = NIL; + + if (SRF_IS_FIRSTCALL()) { + funcctx = SRF_FIRSTCALL_INIT(); + MemoryContext oldcontext = + MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + TupleDesc tupdesc = CreateTemplateTupleDesc(11); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "relid", OIDOID, -1 /* typmod */, + 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "relname", NAMEOID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)3, "last_reader_id", OIDOID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)4, "last_writer_id", OIDOID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)5, "last_read", TIMESTAMPTZOID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)6, "last_write", TIMESTAMPTZOID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)7, "n_select_queries", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)8, "n_insert_queries", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)9, "n_update_queries", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)10, "n_delete_queries", INT8OID, + -1 /* typmod */, 0 /* attdim */); + TupleDescInitEntry(tupdesc, (AttrNumber)11, "n_truncate_queries", INT8OID, + -1 /* typmod */, 0 /* attdim */); + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + StringInfoData dump_file = get_dump_filename(MyDatabaseId); + FILE *dump = AllocateFile(dump_file.data, "rb"); + pfree(dump_file.data); + if (dump) { + while (true) { + relaccessEntry *entry = palloc(sizeof(relaccessEntry)); + if (fread(entry, sizeof(relaccessEntry), 1, dump) != 1) { + pfree(entry); + break; + } + stats_entries = lappend(stats_entries, entry); + } + FreeFile(dump); + } + funcctx->user_fctx = stats_entries; + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + stats_entries = (List *)funcctx->user_fctx; + + while (true) { + if (stats_entries == NIL) { + SRF_RETURN_DONE(funcctx); + } + relaccessEntry *entry = linitial(stats_entries); + stats_entries = list_delete_first(stats_entries); + Datum values[11]; + bool nulls[11]; + MemSet(nulls, 0, sizeof(nulls)); + values[0] = ObjectIdGetDatum(entry->key.relid); + values[1] = CStringGetDatum(entry->relname); + values[2] = ObjectIdGetDatum(entry->last_reader_id); + values[3] = ObjectIdGetDatum(entry->last_writer_id); + values[4] = TimestampTzGetDatum(entry->last_read); + values[5] = TimestampTzGetDatum(entry->last_write); + values[6] = Int64GetDatum(entry->n_select); + values[7] = Int64GetDatum(entry->n_insert); + values[8] = Int64GetDatum(entry->n_update); + values[9] = Int64GetDatum(entry->n_delete); + values[10] = Int64GetDatum(entry->n_truncate); + HeapTuple tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + Datum result = HeapTupleGetDatum(tuple); + funcctx->user_fctx = stats_entries; + /** NOTE: Cannot delete entry from this iteration right now. + * For now let's rely on multi_call_memory_ctx until there is a proven + * memory problem with this codepath + */ + // pfree(entry); + SRF_RETURN_NEXT(funcctx, result); + } +} + +static void relaccess_stats_update_internal() { + LWLockAcquire(data->relaccess_ht_lock, LW_EXCLUSIVE); + relaccess_dump_to_files(true); + LWLockRelease(data->relaccess_ht_lock); + relaccess_upsert_from_file(); +} + +static void add_file_dump_entry(Oid dbid, HTAB *ht) { + bool found; + fileDumpEntry *file_entry = hash_search(ht, &dbid, HASH_ENTER, &found); + if (!found) { + file_entry->dbid = dbid; + StringInfoData filename = get_dump_filename(file_entry->dbid); + file_entry->filename = filename.data; + file_entry->file = AllocateFile(file_entry->filename, "ab"); + } +} + +static void relaccess_dump_to_files(bool only_this_db) { + HTAB *file_mapping; + HASHCTL ctl; + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(fileDumpEntry); + ctl.hash = oid_hash; + file_mapping = hash_create("Relaccess dump files", FILE_CACHE_SZ, &ctl, + HASH_ELEM | HASH_FUNCTION); + LWLockAcquire(data->relaccess_file_lock, LW_EXCLUSIVE); + if (only_this_db) { + add_file_dump_entry(MyDatabaseId, file_mapping); + } else { + HASH_SEQ_STATUS hash_seq; + relaccessEntry *access_entry; + hash_seq_init(&hash_seq, relaccesses); + while ((access_entry = hash_seq_search(&hash_seq)) != NULL) { + add_file_dump_entry(access_entry->key.dbid, file_mapping); + } + } + relaccess_dump_to_files_internal(file_mapping); + HASH_SEQ_STATUS hash_seq; + hash_seq_init(&hash_seq, file_mapping); + fileDumpEntry *entry; + while ((entry = hash_seq_search(&hash_seq)) != NULL) { + FreeFile(entry->file); + pfree(entry->filename); + } + LWLockRelease(data->relaccess_file_lock); + hash_destroy(file_mapping); +} + +static void relaccess_dump_to_files_internal(HTAB *files) { + HASH_SEQ_STATUS hash_seq; + relaccessEntry *entry; + hash_seq_init(&hash_seq, relaccesses); + while ((entry = hash_seq_search(&hash_seq)) != NULL) { + bool found; + fileDumpEntry *dumpfile = + hash_search(files, &entry->key.dbid, HASH_FIND, &found); + if (!found) { + // we don't want to dump events from this DB + continue; + } + if (fwrite(entry, sizeof(relaccessEntry), 1, dumpfile->file) != 1) { + hash_seq_term(&hash_seq); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not write gp_relaccess_stats file \"%s\": %m", + dumpfile->filename))); + break; + } + hash_search(relaccesses, &entry->key, HASH_REMOVE, &found); + had_ht_overflow = false; + } +} + +static void relaccess_upsert_from_file() { + int ret; + if ((ret = SPI_connect()) < 0) { + elog(ERROR, "SPI connect failure - returned %d", ret); + } + LWLockAcquire(data->relaccess_file_lock, LW_EXCLUSIVE); + StringInfoData filename = get_dump_filename(MyDatabaseId); + StringInfoData query; + initStringInfo(&query); + appendStringInfo(&query, + "SELECT relaccess.__relaccess_upsert_from_dump_file()"); + ret = SPI_execute(query.data, false, 1); + unlink(filename.data); + LWLockRelease(data->relaccess_file_lock); + SPI_finish(); + if (ret < 0) { + elog(ERROR, "SPI execute failure - returned %d", ret); + } +} + +static void update_relname_cache(Oid relid, char *relname) { + bool found; + relnameCacheEntry *relname_entry = (relnameCacheEntry *)hash_search( + relname_cache, &relid, HASH_ENTER, &found); + if (!found) { + relname_entry->relid = relid; + if (!relname) { + strlcpy(relname_entry->relname, get_rel_name(relid), + sizeof(relname_entry->relname)); + } else { + strlcpy(relname_entry->relname, relname, sizeof(relname_entry->relname)); + } + } else { + /** + * NOTE: as we don't handle the 'else' clause here, there will be cases when + * we write outdated table names, like below: + * BEGIN; + * INSERT INTO tbl VALUES (1); + * ALTER TABLE tbl RENAME TO new_tbl; + * SELECT * FROM new_tbl; + * COMMIT; + * In this case both INSERT and SELECT stmts would be counted with the + * old'tbl' name, as we don't update our cache for already known relids in + * the same transaction. This is a deliberate decision for performance + * reasons. + */ + } +} + +static void memorize_local_access_entry(Oid relid, AclMode perms) { + bool found; + localAccessKey key; + key.stmt_cnt = stmt_counter; + key.relid = relid; + localAccessEntry *entry = (localAccessEntry *)hash_search( + local_access_entries, &key, HASH_ENTER, &found); + if (!found) { + entry->last_read = entry->last_write = InvalidOid; + entry->perms = perms; + entry->last_read = 0; + entry->last_write = 0; + } else { + entry->perms |= perms; + } + TimestampTz curts = GetCurrentTimestamp(); + if (is_read(perms)) { + entry->last_reader_id = GetUserId(); + entry->last_read = curts; + } + if (is_write(perms)) { + entry->last_writer_id = GetUserId(); + entry->last_write = curts; + } +} + +static void relaccess_executor_end_hook(QueryDesc *query_desc) { + if (prev_ExecutorEnd_hook) { + prev_ExecutorEnd_hook(query_desc); + } else { + standard_ExecutorEnd(query_desc); + } + // Unfortunately, we cannot safely rely on gp_command_counter as + // it is being incremented more than once for many statements. + // So we have to maintain our own statement counter. + stmt_counter++; +} + +static StringInfoData get_dump_filename(Oid dbid) { + StringInfoData filename; + initStringInfoOfSize(&filename, 256); + appendStringInfo(&filename, "%s/relaccess_stats_dump_%d.csv", + PGSTAT_STAT_PERMANENT_DIRECTORY, dbid); + return filename; +} + +static void relaccess_drop_hook(ObjectAccessType access, Oid classId, + Oid objectId, int subId, void *arg) { + if (prev_object_access_hook) { + prev_object_access_hook(access, classId, objectId, subId, arg); + } + // we don't want shared memory and .csv files hanging around forever + // for databases that we've dropped. + // This function cleans up both files and shmem + if (classId == DatabaseRelationId && access == OAT_DROP) { + LWLockAcquire(data->relaccess_ht_lock, LW_EXCLUSIVE); + HASH_SEQ_STATUS hash_seq; + relaccessEntry *entry; + hash_seq_init(&hash_seq, relaccesses); + while ((entry = hash_seq_search(&hash_seq)) != NULL) { + if (entry->key.dbid == objectId) { + bool found; + hash_search(relaccesses, &entry->key, HASH_REMOVE, &found); + had_ht_overflow = false; + } + } + LWLockRelease(data->relaccess_ht_lock); + LWLockAcquire(data->relaccess_file_lock, LW_EXCLUSIVE); + StringInfoData filename = get_dump_filename(objectId); + unlink(filename.data); + pfree(filename.data); + LWLockRelease(data->relaccess_file_lock); + } +} diff --git a/gpcontrib/gp_relaccess_stats/test/expected/gp_relaccess_stats.out b/gpcontrib/gp_relaccess_stats/test/expected/gp_relaccess_stats.out new file mode 100644 index 00000000000..566f313c915 --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/test/expected/gp_relaccess_stats.out @@ -0,0 +1,406 @@ + GP_IGNORE: formatted by atmsort.pm +CREATE EXTENSION gp_relaccess_stats; +-- get rid of NOTICEs +SET client_min_messages TO WARNING; +SET search_path TO relaccess; +DROP TABLE IF EXISTS tbl1 CASCADE; +DROP TABLE IF EXISTS tbl2 CASCADE; +DROP TABLE IF EXISTS tbl3 CASCADE; +DROP TABLE IF EXISTS tbl4 CASCADE; +DROP TABLE IF EXISTS new_tbl1 CASCADE; +DROP TABLE IF EXISTS p3_sales CASCADE; +DROP TABLE IF EXISTS public.last_usr_checks CASCADE; +DROP USER IF EXISTS select_usr; +DROP USER IF EXISTS update_usr; +DROP USER IF EXISTS insert_usr; +DROP USER IF EXISTS delete_usr; +DROP USER IF EXISTS truncate_usr; +-- make sure tracking is ON +SET gp_relaccess_stats.enabled TO 'on'; +SELECT relaccess_stats_init(); + relaccess_stats_init +---------------------- + +(1 row) + +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +TRUNCATE relaccess_stats; +-- test simple actions one by one in separate transactions +CREATE TABLE tbl1 (a INTEGER); +INSERT INTO tbl1 VALUES(1); +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 0 | 1 | 0 | 0 | 0 +(1 row) + +SELECT * FROM tbl1; + a +--- + 1 +(1 row) + +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 1 | 1 | 0 | 0 | 0 +(1 row) + +UPDATE tbl1 SET a = -a; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 2 | 1 | 1 | 0 | 0 +(1 row) + +DELETE FROM tbl1 WHERE a < 0; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 3 | 1 | 1 | 1 | 0 +(1 row) + +TRUNCATE tbl1; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 3 | 1 | 1 | 1 | 1 +(1 row) + +-- verify that rename table works +ALTER TABLE tbl1 RENAME TO new_tbl1; +INSERT INTO new_tbl1 VALUES(1); +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relname = 'tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- +(0 rows) + +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'new_tbl1'::regclass::oid AND relname = 'new_tbl1'; + n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +------------------+------------------+------------------+------------------+-------------------- + 3 | 2 | 1 | 1 | 1 +(1 row) + +TRUNCATE relaccess_stats; +-- multitable truncate +CREATE TABLE tbl1 (a integer); +CREATE TABLE tbl2 (a integer); +TRUNCATE tbl1, tbl2; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats + WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1' OR relid = 'tbl2'::regclass::oid AND relname = 'tbl2' ORDER BY relname; + relname | n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +---------+------------------+------------------+------------------+------------------+-------------------- + tbl1 | 0 | 0 | 0 | 0 | 1 + tbl2 | 0 | 0 | 0 | 0 | 1 +(2 rows) + +TRUNCATE relaccess_stats; +-- test a more complicated statement +CREATE TABLE tbl3 (a integer); +CREATE TABLE tbl4 (a integer); +BEGIN; +-- should give +1 insert for tbl1 and +1 select for other tables +INSERT INTO tbl1 SELECT * FROM tbl2 UNION SELECT * FROM tbl3 UNION SELECT * FROM tbl4; +-- nothing in there before we commit +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT COUNT(*) FROM relaccess_stats WHERE relname LIKE ('tbl_'); + count +------- + 0 +(1 row) + +COMMIT; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries + FROM relaccess_stats WHERE relname LIKE ('tbl_') AND relname::regclass::oid = relid ORDER BY relname; + relname | n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +---------+------------------+------------------+------------------+------------------+-------------------- + tbl1 | 0 | 1 | 0 | 0 | 0 + tbl2 | 1 | 0 | 0 | 0 | 0 + tbl3 | 1 | 0 | 0 | 0 | 0 + tbl4 | 1 | 0 | 0 | 0 | 0 +(4 rows) + +TRUNCATE relaccess_stats; +-- test views +CREATE VIEW v1_2_3 AS (SELECT * FROM tbl2 UNION SELECT * FROM tbl3 UNION SELECT * FROM tbl4); +INSERT INTO tbl1 SELECT * FROM v1_2_3; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries + FROM relaccess_stats WHERE relname = 'v1_2_3' OR relname LIKE ('tbl_') ORDER BY relname; + relname | n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +---------+------------------+------------------+------------------+------------------+-------------------- + tbl1 | 0 | 1 | 0 | 0 | 0 + tbl2 | 1 | 0 | 0 | 0 | 0 + tbl3 | 1 | 0 | 0 | 0 | 0 + tbl4 | 1 | 0 | 0 | 0 | 0 + v1_2_3 | 1 | 0 | 0 | 0 | 0 +(5 rows) + +TRUNCATE relaccess_stats; +-- test timestamps difference +BEGIN; +INSERT INTO tbl1 VALUES (1); +SELECT pg_sleep(1); + pg_sleep +---------- + +(1 row) + +SELECT COUNT(*) FROM tbl1; + count +------- + 1 +(1 row) + +COMMIT; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT EXTRACT(EPOCH FROM (last_read - last_write)) >= 1 FROM relaccess_stats WHERE relname = 'tbl1' AND relid = 'tbl1'::regclass::oid; + ?column? +---------- + t +(1 row) + +TRUNCATE relaccess_stats; +-- test nested partitions lookup +BEGIN; +CREATE TABLE p3_sales (id int, year int, month int, day int, + region text) +DISTRIBUTED BY (id) +PARTITION BY RANGE (year) + SUBPARTITION BY RANGE (month) + SUBPARTITION TEMPLATE ( + START (1) END (13) EVERY (1), + DEFAULT SUBPARTITION other_months ) + SUBPARTITION BY LIST (region) + SUBPARTITION TEMPLATE ( + SUBPARTITION usa VALUES ('usa'), + SUBPARTITION europe VALUES ('europe'), + SUBPARTITION asia VALUES ('asia'), + DEFAULT SUBPARTITION other_regions ) +( START (2002) END (2012) EVERY (1), + DEFAULT PARTITION outlying_years ); +-- 3 inserts into p3_sales root table +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'asia' FROM generate_series(1, 100)i; +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'europe' FROM generate_series(1, 100)i; +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'usa' FROM generate_series(1, 100)i; +-- insert and select to/from specific leaf level partition +INSERT INTO p3_sales_1_prt_11_2_prt_12_3_prt_usa SELECT * FROM p3_sales_1_prt_11_2_prt_12_3_prt_usa; +COMMIT; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries +FROM relaccess_stats WHERE relname LIKE 'p3_sales%' ORDER BY relname; + relname | n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +--------------------------------------+------------------+------------------+------------------+------------------+-------------------- + p3_sales | 0 | 3 | 0 | 0 | 0 + p3_sales_1_prt_11_2_prt_12_3_prt_usa | 1 | 1 | 0 | 0 | 0 +(2 rows) + +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries +FROM relaccess_stats_root_tables_aggregated WHERE relname LIKE 'p3_sales%' ORDER BY relname; + relname | n_select_queries | n_insert_queries | n_update_queries | n_delete_queries | n_truncate_queries +----------+------------------+------------------+------------------+------------------+-------------------- + p3_sales | 1 | 4 | 0 | 0 | 0 +(1 row) + +-- test last_reader and last_writer +CREATE USER select_usr; +CREATE USER update_usr; +CREATE USER insert_usr; +CREATE USER delete_usr; +CREATE USER truncate_usr; +CREATE TABLE public.last_usr_checks(a integer); +GRANT ALL ON TABLE public.last_usr_checks TO select_usr, update_usr, insert_usr, delete_usr, truncate_usr; +SET ROLE select_usr; +SELECT COUNT(*) FROM public.last_usr_checks; + count +------- + 0 +(1 row) + +RESET ROLE; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT (SELECT last_reader_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'select_usr'); + ?column? +---------- + t +(1 row) + +SET ROLE insert_usr; +INSERT INTO public.last_usr_checks VALUES (-1), (0), (1); +RESET ROLE; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'insert_usr'); + ?column? +---------- + t +(1 row) + +SET ROLE update_usr; +UPDATE public.last_usr_checks SET a = a*10 WHERE a < 0; +RESET ROLE; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'update_usr'); + ?column? +---------- + t +(1 row) + +SET ROLE delete_usr; +DELETE FROM public.last_usr_checks WHERE a >= 0; +RESET ROLE; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'delete_usr'); + ?column? +---------- + t +(1 row) + +SET ROLE truncate_usr; +TRUNCATE public.last_usr_checks; +RESET ROLE; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'truncate_usr'); + ?column? +---------- + t +(1 row) + +RESET ROLE; +-- make sure we can turn it OFF +SET gp_relaccess_stats.enabled TO 'off'; +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +TRUNCATE relaccess_stats; +SELECT * FROM tbl1; + a +--- + 1 +(1 row) + +SELECT relaccess_stats_update(); + relaccess_stats_update +------------------------ + +(1 row) + +SELECT count(*) FROM relaccess_stats; + count +------- + 0 +(1 row) + +RESET gp_relaccess_stats.enabled; +DROP TABLE tbl1 CASCADE; +DROP TABLE tbl2 CASCADE; +DROP TABLE tbl3 CASCADE; +DROP TABLE tbl4 CASCADE; +DROP TABLE new_tbl1 CASCADE; +DROP TABLE p3_sales CASCADE; +DROP TABLE public.last_usr_checks CASCADE; +DROP USER select_usr; +DROP USER update_usr; +DROP USER insert_usr; +DROP USER delete_usr; +DROP USER truncate_usr; diff --git a/gpcontrib/gp_relaccess_stats/test/sql/gp_relaccess_stats.sql b/gpcontrib/gp_relaccess_stats/test/sql/gp_relaccess_stats.sql new file mode 100644 index 00000000000..cb96d8eb9fa --- /dev/null +++ b/gpcontrib/gp_relaccess_stats/test/sql/gp_relaccess_stats.sql @@ -0,0 +1,186 @@ +CREATE EXTENSION gp_relaccess_stats; + +-- get rid of NOTICEs +SET client_min_messages TO WARNING; +SET search_path TO relaccess; +DROP TABLE IF EXISTS tbl1 CASCADE; +DROP TABLE IF EXISTS tbl2 CASCADE; +DROP TABLE IF EXISTS tbl3 CASCADE; +DROP TABLE IF EXISTS tbl4 CASCADE; +DROP TABLE IF EXISTS new_tbl1 CASCADE; +DROP TABLE IF EXISTS p3_sales CASCADE; +DROP TABLE IF EXISTS public.last_usr_checks CASCADE; +DROP USER IF EXISTS select_usr; +DROP USER IF EXISTS update_usr; +DROP USER IF EXISTS insert_usr; +DROP USER IF EXISTS delete_usr; +DROP USER IF EXISTS truncate_usr; + +-- make sure tracking is ON +SET gp_relaccess_stats.enabled TO 'on'; +SELECT relaccess_stats_init(); +SELECT relaccess_stats_update(); +TRUNCATE relaccess_stats; + +-- test simple actions one by one in separate transactions +CREATE TABLE tbl1 (a INTEGER); + +INSERT INTO tbl1 VALUES(1); +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + +SELECT * FROM tbl1; +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + +UPDATE tbl1 SET a = -a; +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + +DELETE FROM tbl1 WHERE a < 0; +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + +TRUNCATE tbl1; +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1'; + +-- verify that rename table works +ALTER TABLE tbl1 RENAME TO new_tbl1; +INSERT INTO new_tbl1 VALUES(1); +SELECT relaccess_stats_update(); +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relname = 'tbl1'; +SELECT n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats WHERE relid = 'new_tbl1'::regclass::oid AND relname = 'new_tbl1'; + +TRUNCATE relaccess_stats; +-- multitable truncate +CREATE TABLE tbl1 (a integer); +CREATE TABLE tbl2 (a integer); +TRUNCATE tbl1, tbl2; +SELECT relaccess_stats_update(); +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries FROM relaccess_stats + WHERE relid = 'tbl1'::regclass::oid AND relname = 'tbl1' OR relid = 'tbl2'::regclass::oid AND relname = 'tbl2' ORDER BY relname; + +TRUNCATE relaccess_stats; +-- test a more complicated statement +CREATE TABLE tbl3 (a integer); +CREATE TABLE tbl4 (a integer); + +BEGIN; +-- should give +1 insert for tbl1 and +1 select for other tables +INSERT INTO tbl1 SELECT * FROM tbl2 UNION SELECT * FROM tbl3 UNION SELECT * FROM tbl4; +-- nothing in there before we commit +SELECT relaccess_stats_update(); +SELECT COUNT(*) FROM relaccess_stats WHERE relname LIKE ('tbl_'); +COMMIT; +SELECT relaccess_stats_update(); +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries + FROM relaccess_stats WHERE relname LIKE ('tbl_') AND relname::regclass::oid = relid ORDER BY relname; + +TRUNCATE relaccess_stats; +-- test views +CREATE VIEW v1_2_3 AS (SELECT * FROM tbl2 UNION SELECT * FROM tbl3 UNION SELECT * FROM tbl4); +INSERT INTO tbl1 SELECT * FROM v1_2_3; +SELECT relaccess_stats_update(); +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries + FROM relaccess_stats WHERE relname = 'v1_2_3' OR relname LIKE ('tbl_') ORDER BY relname; + +TRUNCATE relaccess_stats; +-- test timestamps difference +BEGIN; +INSERT INTO tbl1 VALUES (1); +SELECT pg_sleep(1); +SELECT COUNT(*) FROM tbl1; +COMMIT; +SELECT relaccess_stats_update(); +SELECT EXTRACT(EPOCH FROM (last_read - last_write)) >= 1 FROM relaccess_stats WHERE relname = 'tbl1' AND relid = 'tbl1'::regclass::oid; +TRUNCATE relaccess_stats; + +-- test nested partitions lookup +BEGIN; +CREATE TABLE p3_sales (id int, year int, month int, day int, + region text) +DISTRIBUTED BY (id) +PARTITION BY RANGE (year) + SUBPARTITION BY RANGE (month) + SUBPARTITION TEMPLATE ( + START (1) END (13) EVERY (1), + DEFAULT SUBPARTITION other_months ) + SUBPARTITION BY LIST (region) + SUBPARTITION TEMPLATE ( + SUBPARTITION usa VALUES ('usa'), + SUBPARTITION europe VALUES ('europe'), + SUBPARTITION asia VALUES ('asia'), + DEFAULT SUBPARTITION other_regions ) +( START (2002) END (2012) EVERY (1), + DEFAULT PARTITION outlying_years ); +-- 3 inserts into p3_sales root table +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'asia' FROM generate_series(1, 100)i; +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'europe' FROM generate_series(1, 100)i; +INSERT INTO p3_sales SELECT i, i%43+1980, i%12, i%25, 'usa' FROM generate_series(1, 100)i; +-- insert and select to/from specific leaf level partition +INSERT INTO p3_sales_1_prt_11_2_prt_12_3_prt_usa SELECT * FROM p3_sales_1_prt_11_2_prt_12_3_prt_usa; +COMMIT; +SELECT relaccess_stats_update(); +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries +FROM relaccess_stats WHERE relname LIKE 'p3_sales%' ORDER BY relname; +SELECT relname, n_select_queries, n_insert_queries, n_update_queries, n_delete_queries, n_truncate_queries +FROM relaccess_stats_root_tables_aggregated WHERE relname LIKE 'p3_sales%' ORDER BY relname; + +-- test last_reader and last_writer +CREATE USER select_usr; +CREATE USER update_usr; +CREATE USER insert_usr; +CREATE USER delete_usr; +CREATE USER truncate_usr; +CREATE TABLE public.last_usr_checks(a integer); +GRANT ALL ON TABLE public.last_usr_checks TO select_usr, update_usr, insert_usr, delete_usr, truncate_usr; +SET ROLE select_usr; +SELECT COUNT(*) FROM public.last_usr_checks; +RESET ROLE; +SELECT relaccess_stats_update(); +SELECT (SELECT last_reader_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'select_usr'); +SET ROLE insert_usr; +INSERT INTO public.last_usr_checks VALUES (-1), (0), (1); +RESET ROLE; +SELECT relaccess_stats_update(); +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'insert_usr'); +SET ROLE update_usr; +UPDATE public.last_usr_checks SET a = a*10 WHERE a < 0; +RESET ROLE; +SELECT relaccess_stats_update(); +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'update_usr'); +SET ROLE delete_usr; +DELETE FROM public.last_usr_checks WHERE a >= 0; +RESET ROLE; +SELECT relaccess_stats_update(); +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'delete_usr'); +SET ROLE truncate_usr; +TRUNCATE public.last_usr_checks; +RESET ROLE; +SELECT relaccess_stats_update(); +SELECT (SELECT last_writer_id FROM relaccess_stats WHERE RELNAME = 'last_usr_checks') = (SELECT oid FROM pg_roles WHERE rolname = 'truncate_usr'); +RESET ROLE; + +-- make sure we can turn it OFF +SET gp_relaccess_stats.enabled TO 'off'; +SELECT relaccess_stats_update(); +TRUNCATE relaccess_stats; +SELECT * FROM tbl1; +SELECT relaccess_stats_update(); +SELECT count(*) FROM relaccess_stats; +RESET gp_relaccess_stats.enabled; + +DROP TABLE tbl1 CASCADE; +DROP TABLE tbl2 CASCADE; +DROP TABLE tbl3 CASCADE; +DROP TABLE tbl4 CASCADE; +DROP TABLE new_tbl1 CASCADE; +DROP TABLE p3_sales CASCADE; +DROP TABLE public.last_usr_checks CASCADE; +DROP USER select_usr; +DROP USER update_usr; +DROP USER insert_usr; +DROP USER delete_usr; +DROP USER truncate_usr; + diff --git a/pom.xml b/pom.xml index cd19fcd018a..03ea623c0d7 100644 --- a/pom.xml +++ b/pom.xml @@ -1288,7 +1288,11 @@ code or new licensing patterns. gpcontrib/gp_stats_collector/.clang-format gpcontrib/gp_stats_collector/Makefile - contrib/pax_storage/src/test/** From e7554a51e41b324a04ada4ca1592f730045b7b4b Mon Sep 17 00:00:00 2001 From: Aleksey Rozhok Date: Thu, 27 Aug 2026 12:15:24 +0300 Subject: [PATCH 163/167] Feature: add pg_query_state to gp_stats_collector It lets a session expect the live execution state of another running query without waiting for the finish. * add: pgqs tests * add: pgqs source files * add: pgqs kernel patches --- .github/workflows/gpsc-ci.yaml | 316 +++++ .github/workflows/gpsc-crash-test.yaml | 278 ++++ LICENSE | 22 + gpcontrib/gp_stats_collector/Makefile | 3 +- gpcontrib/gp_stats_collector/README.md | 26 + .../docs/pg_query_state_dataflow.puml | 113 ++ .../gp_stats_collector--1.1--1.2.sql | 49 + .../gp_stats_collector--1.2.sql | 159 +++ .../gp_stats_collector.control | 2 +- .../protos/yagpcc_metrics.proto | 65 + .../protos/yagpcc_plan.proto | 67 + .../protos/yagpcc_set_per_node.proto | 93 ++ .../src/PlanNodeEmitter.cpp | 170 +++ .../gp_stats_collector/src/PlanNodeEmitter.h | 45 + .../gp_stats_collector/src/UDSConnector.cpp | 153 ++- .../gp_stats_collector/src/UDSConnector.h | 18 + .../src/gp_stats_collector.c | 9 + .../src/pg_query_state/README.md | 77 ++ .../src/pg_query_state/pg_query_state.c | 1191 +++++++++++++++++ .../src/pg_query_state/pg_query_state.h | 228 ++++ .../src/pg_query_state/qs_types.h | 112 ++ .../src/pg_query_state/signal_handler.c | 1000 ++++++++++++++ gpcontrib/gp_stats_collector/test/Makefile | 22 + .../gp_stats_collector/test/crash/README.md | 38 + .../test/crash/crash_scan.sh | 84 ++ .../test/crash/extract_failures.sh | 37 + .../gp_stats_collector/test/crash/poller.py | 149 +++ .../test/crash/uds_drain.py | 75 ++ .../test/expected/gpsc_pg_query_state.out | 57 + .../test/isolation2/.gitignore | 4 + .../test/isolation2/Makefile | 33 + .../isolation2/expected/gpsc_pqs_backends.out | 27 + .../isolation2/expected/gpsc_pqs_disabled.out | 61 + .../isolation2/expected/gpsc_pqs_perms.out | 92 ++ .../isolation2/expected/gpsc_pqs_running.out | 69 + .../expected/gpsc_pqs_seg_count.out | 58 + .../test/isolation2/expected/setup.out | 6 + .../test/isolation2/isolation2_schedule | 20 + .../test/isolation2/sql/gpsc_pqs_backends.sql | 21 + .../test/isolation2/sql/gpsc_pqs_disabled.sql | 36 + .../test/isolation2/sql/gpsc_pqs_perms.sql | 57 + .../test/isolation2/sql/gpsc_pqs_running.sql | 45 + .../isolation2/sql/gpsc_pqs_seg_count.sql | 36 + .../test/isolation2/sql/setup.sql | 4 + .../test/sql/gpsc_pg_query_state.sql | 46 + pom.xml | 3 + src/backend/commands/explain.c | 156 ++- src/backend/executor/instrument.c | 4 + src/backend/storage/ipc/procsignal.c | 106 ++ src/backend/tcop/postgres.c | 8 +- src/include/commands/explain.h | 2 + src/include/executor/instrument.h | 3 + src/include/storage/procsignal.h | 18 +- 53 files changed, 5543 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/gpsc-ci.yaml create mode 100644 .github/workflows/gpsc-crash-test.yaml create mode 100644 gpcontrib/gp_stats_collector/docs/pg_query_state_dataflow.puml create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql create mode 100644 gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql create mode 100644 gpcontrib/gp_stats_collector/protos/yagpcc_metrics.proto create mode 100644 gpcontrib/gp_stats_collector/protos/yagpcc_plan.proto create mode 100644 gpcontrib/gp_stats_collector/protos/yagpcc_set_per_node.proto create mode 100644 gpcontrib/gp_stats_collector/src/PlanNodeEmitter.cpp create mode 100644 gpcontrib/gp_stats_collector/src/PlanNodeEmitter.h create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/README.md create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h create mode 100644 gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c create mode 100644 gpcontrib/gp_stats_collector/test/Makefile create mode 100644 gpcontrib/gp_stats_collector/test/crash/README.md create mode 100755 gpcontrib/gp_stats_collector/test/crash/crash_scan.sh create mode 100755 gpcontrib/gp_stats_collector/test/crash/extract_failures.sh create mode 100755 gpcontrib/gp_stats_collector/test/crash/poller.py create mode 100755 gpcontrib/gp_stats_collector/test/crash/uds_drain.py create mode 100644 gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/.gitignore create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/Makefile create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql create mode 100644 gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql create mode 100644 gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql diff --git a/.github/workflows/gpsc-ci.yaml b/.github/workflows/gpsc-ci.yaml new file mode 100644 index 00000000000..87e2df5081b --- /dev/null +++ b/.github/workflows/gpsc-ci.yaml @@ -0,0 +1,316 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# gp_stats_collector CI Workflow +# +# Builds Cloudberry with --with-gp-stats-collector (the default in +# configure-cloudberry.sh), stands up a demo cluster with the extension +# preloaded, and runs the gp_stats_collector regression suites. +# +# Scoped to changes that can affect the extension or the core patches it +# depends on, so it does not run on every unrelated push. +# -------------------------------------------------------------------- +name: GPSC CI Pipeline + +on: + push: + branches: + - 'pgqs-**' + pull_request: + paths: + - 'gpcontrib/gp_stats_collector/**' + - '.github/workflows/gpsc-ci.yaml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + + test-gpsc: + name: Build and Test gp_stats_collector (${{ matrix.os }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu22.04 + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + - os: rocky8 + image: apache/incubator-cloudberry:cbdb-build-rocky8-latest + - os: rocky9 + image: apache/incubator-cloudberry:cbdb-build-rocky9-latest + container: + image: ${{ matrix.image }} + options: >- + --user root + -h cdw + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + # Init/build scripts hardcode a "cloudberry" source dir name + # (e.g. create-cloudberry-demo-cluster.sh uses ${SRC_DIR}/../cloudberry), + # so the checkout path must be "cloudberry", not the repo name. + path: cloudberry + submodules: recursive + + - name: Cloudberry Environment Initialization + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin . + chmod -R 755 . + + - name: Configure + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure failed" + exit 1 + fi + + - name: Build + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build failed" + exit 1 + fi + + - name: Create demo cluster (gp_stats_collector preloaded) + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + # pg_query_state requires the module in shared_preload_libraries. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra" + + - name: Run gp_stats_collector regression suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/test-cloudberry.sh + # Capture make output to an artifact file: the raw job log gets + # truncated by the huge Build step, so tail it here on failure. + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + SRC_DIR=${SRC_DIR} \ + PGOPTIONS='' \ + MAKE_NAME='GPSC Regress' \ + MAKE_TARGET=installcheck \ + MAKE_DIRECTORY=--directory=${SRC_DIR}/gpcontrib/gp_stats_collector \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh \ + > ${SRC_DIR}/build-logs/gpsc-regress-make.log 2>&1"; then + echo "::error::gp_stats_collector installcheck failed" + echo "===== gpsc-regress-make.log (tail) =====" + tail -120 ${SRC_DIR}/build-logs/gpsc-regress-make.log 2>/dev/null || true + echo "===== regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Run pg_query_state suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + SRC_DIR=${SRC_DIR} \ + PGOPTIONS='' \ + MAKE_NAME='GPSC pg_query_state' \ + MAKE_TARGET=installcheck \ + MAKE_DIRECTORY=--directory=${SRC_DIR}/gpcontrib/gp_stats_collector/test \ + ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh \ + > ${SRC_DIR}/build-logs/gpsc-pqs-make.log 2>&1"; then + echo "::error::pg_query_state suite failed" + echo "===== gpsc-pqs-make.log (tail) =====" + tail -120 ${SRC_DIR}/build-logs/gpsc-pqs-make.log 2>/dev/null || true + echo "===== test/regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/test/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Upload regression artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpsc-results-${{ matrix.os }} + path: | + cloudberry/gpcontrib/gp_stats_collector/regression.out + cloudberry/gpcontrib/gp_stats_collector/regression.diffs + cloudberry/gpcontrib/gp_stats_collector/results/ + cloudberry/gpcontrib/gp_stats_collector/test/results/ + cloudberry/gpcontrib/gp_stats_collector/test/regression.diffs + cloudberry/build-logs/ + retention-days: 7 + + # Runs the pg_query_state isolation2 suite (multi-session / happy-path checks + # that plain pg_regress cannot express). The fault injector (gp_inject_fault) + # is enabled by default (--enable-faultinjector=yes), so no debug build is + # needed. + test-gpsc-isolation2: + name: pg_query_state multi-session (gp_stats_collector) + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + options: >- + --user root + -h cdw + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + path: cloudberry + submodules: recursive + + - name: Cloudberry Environment Initialization + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin . + chmod -R 755 . + + - name: Configure + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure failed" + exit 1 + fi + + - name: Build + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build failed" + exit 1 + fi + + - name: Build isolation2 harness + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + make -C src/test/isolation2 install"; then + echo "::error::isolation2 harness build failed" + exit 1 + fi + + - name: Create demo cluster (gp_stats_collector preloaded) + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ra" + + - name: Run pg_query_state isolation2 suite + shell: bash + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + run: | + set -eo pipefail + if ! su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + make -C gpcontrib/gp_stats_collector/test/isolation2 installcheck \ + > ${SRC_DIR}/build-logs/gpsc-iso2.log 2>&1"; then + echo "::error::pg_query_state isolation2 suite failed" + echo "===== gpsc-iso2.log (tail) =====" + tail -100 ${SRC_DIR}/build-logs/gpsc-iso2.log 2>/dev/null || true + echo "===== isolation2 regression.diffs =====" + cat ${SRC_DIR}/gpcontrib/gp_stats_collector/test/isolation2/regression.diffs 2>/dev/null || true + exit 1 + fi + + - name: Upload isolation2 artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpsc-iso2-results + path: | + cloudberry/gpcontrib/gp_stats_collector/test/isolation2/results/ + cloudberry/gpcontrib/gp_stats_collector/test/isolation2/regression.diffs + cloudberry/build-logs/ + retention-days: 7 diff --git a/.github/workflows/gpsc-crash-test.yaml b/.github/workflows/gpsc-crash-test.yaml new file mode 100644 index 00000000000..50531f0a97b --- /dev/null +++ b/.github/workflows/gpsc-crash-test.yaml @@ -0,0 +1,278 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# gp_stats_collector crash test +# +# Proves that with the runtime query-state feature fully enabled and a poller +# tracing every running query, Cloudberry does not crash and queries still +# finish with the same results as without the feature. +# +# One build, one demo cluster, two regression passes on it: +# run 1 baseline (feature OFF -- stock Cloudberry) +# run 2 traced (feature ON + poller) + crash gate +# +# Hard verdict: the crash gate (no PANIC / signal / segment down / dead +# coordinator). The failed-test delta (traced \ baseline) is reported for +# information only -- the workload is not diff-deterministic -- and does not +# fail the job. +# +# Workload: make installcheck-parallel (upstream parallel_schedule) -- fast and +# fault-free, so any PANIC in the logs is a genuine crash. +# -------------------------------------------------------------------- +name: GPSC Crash Test + +on: + push: + branches: [REL_2_STABLE] + pull_request: + branches: [REL_2_STABLE] + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + crash-test: + name: installcheck-parallel under tracing + runs-on: ubuntu-latest + container: + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + options: >- + --user root + -h cdw + env: + SRC_DIR: ${{ github.workspace }}/cloudberry + CRASH_DIR: ${{ github.workspace }}/cloudberry/gpcontrib/gp_stats_collector/test/crash + UDS_PATH: /tmp/gpsc_agent.sock + STOP_FILE: /tmp/gpsc_poller.stop + + steps: + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + path: cloudberry + submodules: recursive + + - name: Cloudberry Environment Initialization + shell: bash + run: | + set -eo pipefail + if ! su - gpadmin -c "/tmp/init_system.sh"; then + echo "::error::Container initialization failed" + exit 1 + fi + mkdir -p "${SRC_DIR}/build-logs" + chown -R gpadmin:gpadmin . + chmod -R 755 . + + - name: Configure + shell: bash + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then + echo "::error::Configure failed" + exit 1 + fi + + - name: Build + shell: bash + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then + echo "::error::Build failed" + exit 1 + fi + + - name: Create stock demo cluster + shell: bash + run: | + set -eo pipefail + chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh + if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then + echo "::error::Demo cluster creation failed" + exit 1 + fi + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c optimizer -v on && \ + gpstop -ar" + + - name: 'Run 1: baseline installcheck-parallel (feature OFF)' + shell: bash + run: | + set -eo pipefail + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + make -C src/test/regress installcheck-parallel > ${SRC_DIR}/build-logs/run1-baseline.log 2>&1" || true + cp -f "${SRC_DIR}/src/test/regress/regression.diffs" \ + "${SRC_DIR}/build-logs/run1-baseline.diffs" 2>/dev/null || true + bash "${CRASH_DIR}/extract_failures.sh" "${SRC_DIR}/build-logs/run1-baseline.log" \ + > "${SRC_DIR}/build-logs/baseline-failures.txt" + echo "baseline failures: $(wc -l < ${SRC_DIR}/build-logs/baseline-failures.txt)" + cat "${SRC_DIR}/build-logs/baseline-failures.txt" + + - name: Reset state leaked by the baseline pass + shell: bash + run: | + set -eo pipefail + # installcheck recreates the 'regression' database each pass, but + # CREATE ROLE makes cluster-global roles that outlive it -- run 2's + # test_setup would then fail "role already exists". Drop the baseline + # database (clears role grants/ownership on it) and every regression- + # created role, so run 2 starts from the same clean slate as run 1. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + psql -X -d postgres -c 'DROP DATABASE IF EXISTS regression;' && \ + psql -X -q -A -t -d postgres \ + -c \"SELECT format('DROP ROLE IF EXISTS %I;', rolname) FROM pg_roles WHERE rolname ~ '^(regress|mdb)'\" \ + | psql -X -d postgres -f -" + + - name: Enable full gp_stats_collector config + shell: bash + run: | + set -eo pipefail + # Phase 1: load the module, then restart so its custom GUCs are known. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ + gpstop -ar && \ + sleep 10" + # Phase 2: enable every logging/polling knob, then restart again. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + gpconfig -c pg_query_state.enable -v on && \ + gpconfig -c pg_query_state.enable_timing -v on && \ + gpconfig -c pg_query_state.enable_buffers -v on && \ + gpconfig -c gpsc.enable -v on && \ + gpconfig -c gpsc.enable_analyze -v on && \ + gpconfig -c gpsc.enable_cdbstats -v on && \ + gpconfig -c gpsc.report_nested_queries -v on && \ + gpconfig -c gpsc.logging_mode -v UDS && \ + gpconfig -c gpsc.uds_path -v ${UDS_PATH} && \ + gpconfig -c compute_query_id -v regress && \ + gpstop -ar && \ + sleep 10" + + - name: Install and smoke-test extension + shell: bash + run: | + set -eo pipefail + # The poller connects to 'postgres' and calls gpsc.pg_query_state; that + # SQL entry point only exists where the extension is created. Without + # this the traced run would be vacuous (every poll would just error on + # a missing function), so assert the function is resolvable and fail + # loudly if it is not. + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + psql -X -d postgres -c 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector;' && \ + psql -X -q -A -t -d postgres \ + -c \"SELECT 'gpsc.pg_query_state(int,bytea)'::regprocedure;\"" \ + || { echo "::error::gpsc.pg_query_state not resolvable -- extension not installed; traced run would be vacuous"; exit 1; } + + - name: Start UDS drain + shell: bash + run: | + set -eo pipefail + chown -R gpadmin:gpadmin "${CRASH_DIR}" + su - gpadmin -c "nohup python3 ${CRASH_DIR}/uds_drain.py --path ${UDS_PATH} \ + > ${SRC_DIR}/build-logs/uds-drain.log 2>&1 &" + sleep 2 + test -S "${UDS_PATH}" || { echo "::error::UDS drain socket not created"; exit 1; } + + - name: 'Run 2: traced installcheck-parallel (poller running)' + shell: bash + run: | + set -eo pipefail + rm -f "${STOP_FILE}" + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + nohup python3 ${CRASH_DIR}/poller.py --stop-file ${STOP_FILE} \ + > ${SRC_DIR}/build-logs/poller.log 2>&1 &" + su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + make -C src/test/regress installcheck-parallel > ${SRC_DIR}/build-logs/run2-traced.log 2>&1" || true + touch "${STOP_FILE}" + sleep 3 + cp -f "${SRC_DIR}/src/test/regress/regression.diffs" \ + "${SRC_DIR}/build-logs/run2-traced.diffs" 2>/dev/null || true + bash "${CRASH_DIR}/extract_failures.sh" "${SRC_DIR}/build-logs/run2-traced.log" \ + > "${SRC_DIR}/build-logs/traced-failures.txt" + echo "traced failures: $(wc -l < ${SRC_DIR}/build-logs/traced-failures.txt)" + cat "${SRC_DIR}/build-logs/traced-failures.txt" + + - name: 'Crash gate (hard verdict)' + shell: bash + run: | + set -eo pipefail + if ! su - gpadmin -c "cd ${SRC_DIR} && \ + source /usr/local/cloudberry-db/cloudberry-env.sh && \ + source gpAux/gpdemo/gpdemo-env.sh && \ + bash ${CRASH_DIR}/crash_scan.sh ${SRC_DIR}/gpAux/gpdemo/datadirs \ + > ${SRC_DIR}/build-logs/crash-scan.log 2>&1"; then + echo "::error::Crash gate tripped -- Cloudberry did not survive tracing" + cat "${SRC_DIR}/build-logs/crash-scan.log" + exit 1 + fi + cat "${SRC_DIR}/build-logs/crash-scan.log" + + - name: 'Failed-test delta (informational)' + shell: bash + run: | + set -eo pipefail + # Tests whose tracing diff is client-message noise, not a correctness + # signal. strings: QD parse-time WARNINGs ("nonstandard use of \\", + # scan.l escape_string_warning) re-emit non-deterministically when the + # poller ProcSignal lands mid-statement; the query has no runtime stats. + printf '%s\n' strings | sort -u > "${SRC_DIR}/build-logs/known-flaky.txt" + # baseline/traced-failures.txt are already sort -u (extract_failures.sh). + comm -13 \ + "${SRC_DIR}/build-logs/baseline-failures.txt" \ + "${SRC_DIR}/build-logs/traced-failures.txt" \ + | comm -23 - "${SRC_DIR}/build-logs/known-flaky.txt" \ + > "${SRC_DIR}/build-logs/delta.txt" + count=$(wc -l < "${SRC_DIR}/build-logs/delta.txt") + echo "tests failing under tracing but not in the stock baseline: ${count}" + cat "${SRC_DIR}/build-logs/delta.txt" + if [ "${count}" -gt 0 ]; then + echo "::warning::${count} test(s) failed only under tracing (informational; the workload is not diff-deterministic -- inspect run2-traced.diffs)." + fi + + - name: Upload crash-test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: gpsc-crash-test-results + path: | + cloudberry/build-logs/ + retention-days: 14 diff --git a/LICENSE b/LICENSE index 0ccd7072122..7aaffa76495 100644 --- a/LICENSE +++ b/LICENSE @@ -200,6 +200,28 @@ See the License for the specific language governing permissions and limitations under the License. +================================================================================ +This product includes software derived from pg_query_state +(https://github.com/postgrespro/pg_query_state), under the PostgreSQL License: + + Copyright (c) 2016-2025, Postgres Professional + + Permission to use, copy, modify, and distribute this software and its + documentation for any purpose, without fee, and without a written agreement + is hereby granted, provided that the above copyright notice and this + paragraph and the following two paragraphs appear in all copies. + + IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT, + INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST + PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN + IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT + NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, + AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, + UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + ================================================================================ This product includes software from PostgreSQL, under the PostgreSQL License: diff --git a/gpcontrib/gp_stats_collector/Makefile b/gpcontrib/gp_stats_collector/Makefile index b3228d2c45e..7c8e2b269af 100644 --- a/gpcontrib/gp_stats_collector/Makefile +++ b/gpcontrib/gp_stats_collector/Makefile @@ -3,7 +3,7 @@ EXTENSION = gp_stats_collector DATA = $(wildcard *--*.sql) REGRESS = gpsc_cursors gpsc_dist gpsc_select gpsc_utf8_trim gpsc_utility gpsc_guc_cache gpsc_uds gpsc_locale -PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service +PROTO_BASES = gpsc_plan gpsc_metrics gpsc_set_service yagpcc_metrics yagpcc_plan yagpcc_set_per_node PROTO_OBJS = $(patsubst %,src/protos/%.pb.o,$(PROTO_BASES)) C_OBJS = $(patsubst %.c,%.o,$(wildcard src/*.c src/*/*.c)) @@ -11,6 +11,7 @@ CPP_OBJS = $(patsubst %.cpp,%.o,$(wildcard src/*.cpp src/log/*.cpp src/memory/*. OBJS = $(C_OBJS) $(CPP_OBJS) $(PROTO_OBJS) PG_CXXFLAGS += -Werror -Wall -Wno-unused-but-set-variable -std=c++17 -Isrc/protos -Isrc -Iinclude -DGPBUILD +PG_CPPFLAGS += -I$(libpq_srcdir) -Isrc/protos -Isrc -Iinclude SHLIB_LINK += -lprotobuf -lstdc++ EXTRA_CLEAN = src/protos diff --git a/gpcontrib/gp_stats_collector/README.md b/gpcontrib/gp_stats_collector/README.md index 8c2d5c6868e..5b1ac0d1b25 100644 --- a/gpcontrib/gp_stats_collector/README.md +++ b/gpcontrib/gp_stats_collector/README.md @@ -45,3 +45,29 @@ An extension for collecting query execution metrics and reporting them to an ext - **User Filtering:** To exclude activity from certain roles, add them to the comma-separated list in `gpsc.ignored_users_list`. - **Trimming plans:** Query texts and execution plans are trimmed based on `gpsc.max_text_size` and `gpsc.max_plan_size` (default: 1024KB). For now, it is not recommended to set these GUCs higher than 1024KB. - **Analyze collection:** Analyze is sent if execution time exceeds `gpsc.min_analyze_time`, which is 10 seconds by default. Analyze is collected if `gpsc.enable_analyze` is true. + +### Runtime Query State (`pg_query_state`) + +On-demand inspection of the live execution state of another running backend. The target's active plan tree is walked across the coordinator (QD) and every segment (QE), collecting per-node instrumentation, without waiting for the query to finish. Each backend pushes its own snapshot to the UDS sink configured by `gpsc.uds_path`, keyed by the caller-supplied `trace_id`. + +Delivery is best-effort, exactly like the rest of the extension: a snapshot that does not fit into the socket is dropped rather than retried, so a slow or absent reader never adds latency to the query being observed. + +The functions live in the `gpsc` schema (extension version 1.2). + +#### 1. `pg_query_state(pid, trace_id)` +- **What:** Triggers runtime per-node collection for the query running on backend `pid`. Fans a poll out to every participating QE and to the QD; each backend walks its plan tree and pushes one per-node batch. The coordinator additionally pushes the deparsed plan document, rate-limited so that repeated polls of a long query do not resend an unchanged plan. Fire-and-forget: returns `void`. +- **Arguments:** `trace_id` is a `bytea` of exactly 16 bytes, minted by the caller and used as the collection key on the receiving side. +- **Executes on:** the coordinator only. +- **GUC:** `pg_query_state.enable`. + +#### 2. `pg_query_state_backends(pid)` +- **What:** Lists the QE backends participating in the query running on backend `pid`, as `(segid, pid)` rows, so that a collector knows how many batches to expect. A coordinator-only query (`INSERT ... VALUES`, catalog reads) allocates no gang, and is reported as a single row for the coordinator itself with `segid < 0`. Returns an empty set when the target is not running a query or has the module disabled. +- **GUC:** `pg_query_state.enable`. + +#### 3. `cbdb_mpp_query_state(gp_segment_pid[], trace_id)` +- **What:** QE-side dispatch target used internally by `pg_query_state()`; not intended for direct use. + +### Runtime Query State Configuration +- **Enable:** `pg_query_state.enable` (default `on`) turns the executor hooks and signal handling on or off. Additional GUCs `pg_query_state.enable_timing` and `pg_query_state.enable_buffers` control the level of instrumentation collected. +- **Permissions:** The functions are granted to `PUBLIC`, but access is checked in the server: a caller may poll a backend only if it is a superuser or owns the target query. This lets monitoring agents run under a non-superuser role while still preventing one role from observing another's queries. +- **Preload:** The module registers custom signal handlers at startup, so `gp_stats_collector` must be listed in `shared_preload_libraries`. diff --git a/gpcontrib/gp_stats_collector/docs/pg_query_state_dataflow.puml b/gpcontrib/gp_stats_collector/docs/pg_query_state_dataflow.puml new file mode 100644 index 00000000000..bb0b2bc2340 --- /dev/null +++ b/gpcontrib/gp_stats_collector/docs/pg_query_state_dataflow.puml @@ -0,0 +1,113 @@ +' Licensed to the Apache Software Foundation (ASF) under one +' or more contributor license agreements. See the NOTICE file +' distributed with this work for additional information +' regarding copyright ownership. The ASF licenses this file +' to you under the Apache License, Version 2.0 (the +' "License"); you may not use this file except in compliance +' with the License. You may obtain a copy of the License at +' +' http://www.apache.org/licenses/LICENSE-2.0 +' +' Unless required by applicable law or agreed to in writing, software +' distributed under the License is distributed on an "AS IS" BASIS, +' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and +' limitations under the License. +' +' pg_query_state runtime per-node stats -- data flow. +' Render: plantuml docs/pg_query_state_dataflow.puml (produces a PNG/SVG) +' +' Key point of this diagram: the SQL functions run on their OWN (requestor) +' backends; the query being observed lives in SEPARATE (target) backends on the +' same host, reached only by ProcSignal. + +@startuml pg_query_state_dataflow +title pg_query_state — on-demand runtime per-node stats (keyed by trace_id) + +skinparam backgroundColor #FFFFFF +skinparam shadowing false +skinparam roundcorner 8 +skinparam sequence { + ArrowThickness 1.4 + LifeLineBorderColor #9AA5B1 + LifeLineBackgroundColor #F5F7FA + ParticipantBorderColor #52606D + ParticipantBackgroundColor #E4E7EB + ParticipantFontStyle bold + BoxBorderColor #B8C1CC + NoteBackgroundColor #FFF9E6 + NoteBorderColor #E0C879 +} + +' Arrow colour convention (see legend): +' blue = SQL / MPP dispatch (libpq) +' red = ProcSignal (custom signal), same host +' gray = shared-memory queue (shm_mq) reply +' green = protobuf over Unix domain socket (UDS) + +actor UI as "UI" +participant M as "yagpcc\n(master)" + +box "Coordinator host" #F0F4F8 + participant RQD as "Requestor QD\nruns pg_query_state()" + participant TQD as "Target QD\n(observed query)" +end box + +box "Segment hosts" #F0F4F8 + collections RQE as "Requestor QEs\nrun cbdb_mpp_query_state()" + collections TQE as "Target QEs\n(observed query)" +end box + +participant Y as "Local yagpcc\n(per-host UDS sink)" + +autonumber + +UI -[#2F80ED]> M : POST /api/per-node-stats?pid +hnote over M #EAF2FF : mint **trace_id**\n(16 raw bytes, UUID) + +== Expected backend set (completeness barrier) == +M -[#2F80ED]> RQD : gpsc.pg_query_state_backends(pid) +RQD -[#EB5757]> TQD : BackendInfoPollReason (signal) +TQD -[#828282]-> RQD : active QE set via shm_mq\n(SendCdbComponents runs in TQD) +RQD --[#2F80ED]> M : expected backend set (segid, pid) + +== Fan out the poll == +M -[#2F80ED]> RQD : gpsc.pg_query_state(pid, **trace_id**) +note over RQD + validate **trace_id** (exactly 16 B) + permission gate: + superuser() || GetUserId() == proc->roleId + resolve target QEs (same BackendInfoPollReason round-trip) +end note + +note over RQD, TQD : RQD stamps qs_trace_slots[**TQD** backendId] = **trace_id** +RQD -[#EB5757]> TQD : SendProcSignal(QueryStatePollReason) + +RQD -[#2F80ED]> RQE : CdbDispatchCommand:\ncbdb_mpp_query_state(seg_pid[], **trace_id**)\n(a separate requestor backend per segment host) +note over RQE, TQE : each RQE stamps qs_trace_slots[**TQE** backendId] = **trace_id** +RQE -[#EB5757]> TQE : SendProcSignal(QueryStatePollReason) + +== Collect (in the target backends' signal handler) == +note over TQD, TQE + SendQueryState(): walk the LIVE plan tree, + one GpscNodeSample per plan node +end note +TQD -[#27AE60]> Y : SetPerNodeBatchReq (**trace_id**) +TQE -[#27AE60]> Y : SetPerNodeBatchReq (**trace_id**) +TQD -[#27AE60]> Y : SetQueryPlanReq (QD only, rate-limited) + +== Pull and pivot == +M -[#2F80ED]> Y : pull batches for **trace_id**\n(every segment, concurrent) +Y --[#27AE60]> M : per-node batches +hnote over M #EAF2FF : fold QD copy,\npivot flat samples\ninto a per-slice tree +M --[#2F80ED]> UI : slices[] tree (JSON)\n(+ plan-doc via /api/per-node-plan) + +legend right + |= arrow |= transport | + | —— | SQL / MPP dispatch (libpq) | + | —— | ProcSignal (same host) | + | —— | shm_mq reply | + | —— | protobuf over UDS | +endlegend + +@enduml diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql new file mode 100644 index 00000000000..ce0659d5032 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.1--1.2.sql @@ -0,0 +1,49 @@ +/* gp_stats_collector--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_stats_collector UPDATE TO '1.2'" to load this file. \quit + +-- Compact (segid, pid) identifier for a QE backend running on a segment. +-- Matches the C gp_segment_pid struct used by the pg_query_state signal layer. +CREATE TYPE gpsc.gp_segment_pid AS ( + segid int, + pid int +); + +-- pg_query_state(pid): trigger runtime per-node collection for the query +-- running on backend `pid`. Fans QueryStatePollReason out to every QE via +-- cbdb_mpp_query_state; each matching QE walks its plan tree and pushes a +-- per-node batch to its local yagpcc over UDS. Fire-and-forget: returns void. +CREATE FUNCTION gpsc.pg_query_state(pid int, trace_id bytea) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'pg_query_state' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- cbdb_mpp_query_state(gp_segment_pid[], trace_id): dispatched verbatim to +-- every segment by pg_query_state() via CdbDispatchCommand; runs locally on +-- each QE, so no EXECUTE ON marker. Signals the matching local backends. The +-- trace_id is stamped into every per-node batch so all backends' pushes land +-- under the one key this collection owns. +CREATE FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], trace_id bytea) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'cbdb_mpp_query_state' +LANGUAGE C VOLATILE; + +-- pg_query_state_backends(pid): list the QE backends participating in the +-- query running on backend `pid`, as (segid, pid) rows. yagpcc uses the row +-- count as the "expected batches" barrier: per-node collection is complete +-- once a batch has arrived from every listed backend. +CREATE FUNCTION gpsc.pg_query_state_backends(pid int) +RETURNS TABLE(segid int, pid int) +AS 'MODULE_PATHNAME', 'pg_query_state_backends' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- The runtime query-state API is callable by any role; the per-backend +-- permission gate in C (superuser or the query's owner) enforces access, so +-- these can be granted broadly. This lets monitoring agents (e.g. yagpcc) run +-- under a non-superuser role. cbdb_mpp_query_state is dispatched to the QEs +-- under the caller's role, so it needs EXECUTE too. +GRANT USAGE ON SCHEMA gpsc TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state(int, bytea) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state_backends(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], bytea) TO PUBLIC; \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql b/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql new file mode 100644 index 00000000000..8e3bbeeae88 --- /dev/null +++ b/gpcontrib/gp_stats_collector/gp_stats_collector--1.2.sql @@ -0,0 +1,159 @@ +/* gp_stats_collector--1.2.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_stats_collector" to load this file. \quit + +CREATE SCHEMA gpsc; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_reset_f_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_stat_messages_reset' +LANGUAGE C EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.stat_messages_reset() +RETURNS SETOF void +AS +$$ + SELECT gpsc.__stat_messages_reset_f_on_master(); + SELECT gpsc.__stat_messages_reset_f_on_segments(); +$$ +LANGUAGE SQL EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_f_on_master() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__stat_messages_f_on_segments() +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'gpsc_stat_messages' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE VIEW gpsc.stat_messages AS + SELECT C.* + FROM gpsc.__stat_messages_f_on_master() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) + UNION ALL + SELECT C.* + FROM gpsc.__stat_messages_f_on_segments() as C ( + segid int, + total_messages bigint, + send_failures bigint, + connection_failures bigint, + other_errors bigint, + max_message_size int + ) +ORDER BY segid; + +CREATE FUNCTION gpsc.__init_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__init_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_init_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +-- Creates log table inside gpsc schema. +SELECT gpsc.__init_log_on_master(); +SELECT gpsc.__init_log_on_segments(); + +CREATE VIEW gpsc.log AS + SELECT * FROM gpsc.__log -- master + UNION ALL + SELECT * FROM gp_dist_random('gpsc.__log') -- segments +ORDER BY tmid, ssid, ccnt; + +CREATE FUNCTION gpsc.__truncate_log_on_master() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__truncate_log_on_segments() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_truncate_log' +LANGUAGE C STRICT VOLATILE EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION gpsc.truncate_log() +RETURNS SETOF void AS $$ +BEGIN + PERFORM gpsc.__truncate_log_on_master(); + PERFORM gpsc.__truncate_log_on_segments(); +END; +$$ LANGUAGE plpgsql VOLATILE; + +CREATE FUNCTION gpsc.__test_uds_start_server(path text) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_start_server' +LANGUAGE C STRICT EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__test_uds_receive(timeout_ms int DEFAULT 2000) +RETURNS SETOF bigint +AS 'MODULE_PATHNAME', 'gpsc_test_uds_receive' +LANGUAGE C STRICT EXECUTE ON COORDINATOR; + +CREATE FUNCTION gpsc.__test_uds_stop_server() +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'gpsc_test_uds_stop_server' +LANGUAGE C EXECUTE ON COORDINATOR; + +-- --------------------------------------------------------------------------- +-- 1.2: pg_query_state per-node runtime collection (push to yagpcc via UDS) +-- --------------------------------------------------------------------------- + +-- Compact (segid, pid) identifier for a QE backend running on a segment. +-- Matches the C gp_segment_pid struct used by the pg_query_state signal layer. +CREATE TYPE gpsc.gp_segment_pid AS ( + segid int, + pid int +); + +-- pg_query_state(pid): trigger runtime per-node collection for the query +-- running on backend `pid`. Fans QueryStatePollReason out to every QE via +-- cbdb_mpp_query_state; each matching QE walks its plan tree and pushes a +-- per-node batch to its local yagpcc over UDS. Fire-and-forget: returns void. +CREATE FUNCTION gpsc.pg_query_state(pid int, trace_id bytea) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'pg_query_state' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- cbdb_mpp_query_state(gp_segment_pid[], trace_id): dispatched verbatim to +-- every segment by pg_query_state() via CdbDispatchCommand; runs locally on +-- each QE, so no EXECUTE ON marker. Signals the matching local backends. The +-- trace_id is stamped into every per-node batch so all backends' pushes land +-- under the one key this collection owns. +CREATE FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], trace_id bytea) +RETURNS SETOF void +AS 'MODULE_PATHNAME', 'cbdb_mpp_query_state' +LANGUAGE C VOLATILE; + +-- pg_query_state_backends(pid): list the QE backends participating in the +-- query running on backend `pid`, as (segid, pid) rows. yagpcc uses the row +-- count as the "expected batches" barrier: per-node collection is complete +-- once a batch has arrived from every listed backend. +CREATE FUNCTION gpsc.pg_query_state_backends(pid int) +RETURNS TABLE(segid int, pid int) +AS 'MODULE_PATHNAME', 'pg_query_state_backends' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; + +-- The runtime query-state API is callable by any role; the per-backend +-- permission gate in C (superuser or the query's owner) enforces access, so +-- these can be granted broadly. This lets monitoring agents (e.g. yagpcc) run +-- under a non-superuser role. cbdb_mpp_query_state is dispatched to the QEs +-- under the caller's role, so it needs EXECUTE too. +GRANT USAGE ON SCHEMA gpsc TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state(int, bytea) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.pg_query_state_backends(int) TO PUBLIC; +GRANT EXECUTE ON FUNCTION gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], bytea) TO PUBLIC; \ No newline at end of file diff --git a/gpcontrib/gp_stats_collector/gp_stats_collector.control b/gpcontrib/gp_stats_collector/gp_stats_collector.control index 4aea2bd49b8..76cf6c26e2b 100644 --- a/gpcontrib/gp_stats_collector/gp_stats_collector.control +++ b/gpcontrib/gp_stats_collector/gp_stats_collector.control @@ -1,5 +1,5 @@ # gp_stats_collector extension comment = 'Intercept query and plan execution hooks and report them to Cloudberry monitor agents' -default_version = '1.1' +default_version = '1.2' module_pathname = '$libdir/gp_stats_collector' superuser = true diff --git a/gpcontrib/gp_stats_collector/protos/yagpcc_metrics.proto b/gpcontrib/gp_stats_collector/protos/yagpcc_metrics.proto new file mode 100644 index 00000000000..d51da308e44 --- /dev/null +++ b/gpcontrib/gp_stats_collector/protos/yagpcc_metrics.proto @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package yagpcc; + +/* + * Instrumentation counters for a single plan node execution. + * + * Field semantics mirror the PostgreSQL Instrumentation struct: + * ntuples -- total tuples produced (completed loops) + * nloops -- number of completed execution loops + * tuplecount -- tuples emitted so far in the current (in-progress) loop + * firsttuple -- wall time to first tuple of this cycle (seconds) + * startup -- total startup time across all loops (seconds) + * total -- total elapsed time across all loops (seconds) + */ +message MetricInstrumentation { + uint64 ntuples = 1; + uint64 nloops = 2; + uint64 tuplecount = 3; + double firsttuple = 4; + double startup = 5; + double total = 6; + uint64 shared_blks_hit = 7; + uint64 shared_blks_read = 8; +} + +/* + * Node-level metrics container. Currently wraps only instrumentation; may + * be extended with system/spill stats in future revisions. + */ +message NodeMetrics { + MetricInstrumentation instrumentation = 1; +} + +/* + * Common query and segment identification keys reused across messages. + */ +message QueryKey { + int32 tmid = 1; /* gp_gettmid() transaction/time identifier */ + int32 ssid = 2; /* gp_session_id */ + int32 ccnt = 3; /* gp_command_count */ +} + +message SegmentKey { + int32 dbid = 1; /* GpIdentity.dbid */ + int32 segindex = 2; /* GpIdentity.segindex (-1 = coordinator) */ +} diff --git a/gpcontrib/gp_stats_collector/protos/yagpcc_plan.proto b/gpcontrib/gp_stats_collector/protos/yagpcc_plan.proto new file mode 100644 index 00000000000..3d38b5a50f7 --- /dev/null +++ b/gpcontrib/gp_stats_collector/protos/yagpcc_plan.proto @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +import "protos/yagpcc_metrics.proto"; + +package yagpcc; + +message SetQueryPlanReq { + google.protobuf.Timestamp datetime = 1; + QueryKey query_key = 2; + string plan_doc = 3; /* ExplainPrintPlan output */ + int32 format = 4; /* ExplainFormat: 0=text 1=xml 2=json 3=yaml */ +} + +/* + * Execution status of a single plan node. + * + * Mirrors QsNodeStatus from qs_types.h: + * INITIALIZED -- node was set up but has not yet started execution + * EXECUTING -- node is currently inside a tuple-fetch call + * FINISHED -- node has completed at least one full execution loop + */ +enum PlanNodeStatus { + PLAN_NODE_STATUS_UNSPECIFIED = 0; + PLAN_NODE_STATUS_INITIALIZED = 1; + PLAN_NODE_STATUS_EXECUTING = 2; + PLAN_NODE_STATUS_FINISHED = 3; +} + +/* + * Identifying information for a single plan node within a query plan tree. + * + * Fields: + * plan_node_id -- unique node id within the plan (Plan.plan_node_id) + * parent_plan_node_id -- plan_node_id of the logical parent node, or 0 + * node_type -- PostgreSQL NodeTag value (nodeTag(plan)) + * slice_id -- CDB slice index (currentSliceId) + * plan_rows -- optimizer row-count estimate (Plan.plan_rows) + * relation_oid -- OID of the scanned relation for scan nodes, or 0 + */ +message PlanNode { + int32 plan_node_id = 1; + int32 parent_plan_node_id = 2; + int32 node_type = 3; + int32 slice_id = 4; + double plan_rows = 5; + int32 relation_oid = 6; +} diff --git a/gpcontrib/gp_stats_collector/protos/yagpcc_set_per_node.proto b/gpcontrib/gp_stats_collector/protos/yagpcc_set_per_node.proto new file mode 100644 index 00000000000..86746b2e4c4 --- /dev/null +++ b/gpcontrib/gp_stats_collector/protos/yagpcc_set_per_node.proto @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +import "protos/yagpcc_metrics.proto"; +import "protos/yagpcc_plan.proto"; + +package yagpcc; + +/* + * SetPerNodeBatchReq -- one whole plan-tree snapshot from a single backend. + * + * Sent once per walker pass (SendQueryState signal or pg_qs_executor_end), + * carrying every observed plan node in one message. A backend opens one UDS + * connection and sends this batch instead of connect+send+close per node. The + * shared query_key, segment_key and datetime are hoisted out of every node. + * + * Wire transport: UDSConnector::report_per_node_batch() with the 8-byte + * extended protocol header (payload_size | 0x80000000, request_type=1, + * reserved=0). + * + * trace_id keys the whole collection: 16 raw bytes minted once on the + * coordinator per pg_query_state() call and stamped into every backend's batch, + * so yagpcc can group the snapshots of one poll and tell them apart from an + * overlapping poll of the same pid. + * + * This message MUST stay byte-identical to the yagpcc-side definition in + * api/proto/agent_segment/yagpcc_set_service.proto. + */ +message SetPerNodeBatchReq { + google.protobuf.Timestamp datetime = 1; + SegmentKey segment_key = 2; + repeated BatchNode nodes = 3; + bytes trace_id = 4; /* 16-byte per-collection key (see above) */ +} + +/* + * BatchNode -- one plan node inside a SetPerNodeBatchReq. + * + * Deliberately flat (no NodeMetrics wrapper) so the two repos can keep the + * message trivially wire-identical without sharing a metrics wrapper type. + * Fields mirror GpscNodeSample minus the hoisted identity keys. + */ +message BatchNode { + int32 plan_node_id = 1; + int32 parent_plan_node_id = 2; + int32 node_type = 3; /* raw NodeTag value */ + int32 slice_id = 4; + double plan_rows = 5; /* planner estimate */ + int32 relation_oid = 6; /* scan relation OID, 0 otherwise */ + double ntuples = 7; + double tuplecount = 8; + double nloops = 9; + double startup = 10; + double total = 11; + double firsttuple = 12; + uint64 shared_blks_hit = 13; + uint64 shared_blks_read = 14; + PlanNodeStatus node_status = 15; + bool eof = 16; + google.protobuf.Timestamp executed_at = 17; + bool workfile_created = 18; + int64 workmem_used = 19; /* bytes of work_mem actually used */ + int64 workmem_wanted = 20; /* bytes needed to avoid spill; >0 == spilled */ + double ntuples_delta = 21; /* tuples produced since the previous sample */ + double tuples_per_sec = 22; /* ntuples_delta over the sample interval */ + double time_since_init_sec = 23; /* seconds since the node's first sample */ + bool stalled = 24; /* executing, no new tuples, not at eof */ + int32 pid = 25; + reserved 26; + reserved "segindex"; + string relation_name = 27; + int32 ccnt = 28; +} + diff --git a/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.cpp b/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.cpp new file mode 100644 index 00000000000..4e299e3453a --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.cpp @@ -0,0 +1,170 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PlanNodeEmitter.cpp + * Build and send per-node protobuf messages to the yagpcc UDS sink. + * + * This file is the bridge between the C pg_query_state layer and the C++ + * protobuf / UDS connector infrastructure. It implements the functions + * declared in PlanNodeEmitter.h and callable from plain C: + * + * gpsc_qs_sync_config() -- reload the Config singleton + * gpsc_emit_node_batch() -- serialize a plan-tree snapshot and send it + * gpsc_emit_query_plan() -- serialize a plan document and send it + * + * The outgoing message types are yagpcc::SetPerNodeBatchReq and + * yagpcc::SetQueryPlanReq (generated from protos/yagpcc_set_per_node.proto). + * Transmission is handled by UDSConnector, which prepends the 8-byte extended + * protocol header before writing to the socket. + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/PlanNodeEmitter.cpp + */ + +#include "PlanNodeEmitter.h" +#include "protos/yagpcc_set_per_node.pb.h" +#include "UDSConnector.h" +#include "Config.h" +#include "ProtoUtils.h" + +/* Module-private Config instance shared across all emit calls in a session. */ +static Config pne_config; + +/* + * gpsc_qs_sync_config -- reload the Config singleton. + * + * Must be called before a gpsc_emit_node_batch() call so that the UDS path + * and other settings are up to date. It is a no-op when the config has not + * changed since the last call. + */ +extern "C" void +gpsc_qs_sync_config() +{ + pne_config.sync(); +} + +/* + * map_node_status -- convert a QsNodeStatus enum to yagpcc::PlanNodeStatus. + * + * Returns PLAN_NODE_STATUS_UNSPECIFIED for any value not recognised by the + * switch, which is safe because the receiver ignores unknown status codes. + */ +static yagpcc::PlanNodeStatus +map_node_status(QsNodeStatus status) +{ + switch (status) + { + case QS_NODE_STATUS_INITIALIZED: + return yagpcc::PLAN_NODE_STATUS_INITIALIZED; + case QS_NODE_STATUS_EXECUTING: + return yagpcc::PLAN_NODE_STATUS_EXECUTING; + case QS_NODE_STATUS_FINISHED: + return yagpcc::PLAN_NODE_STATUS_FINISHED; + default: + return yagpcc::PLAN_NODE_STATUS_UNSPECIFIED; + } +} + +extern "C" void +gpsc_emit_node_batch(GpscNodeSample **nodes, int count, const char *trace_id) +{ + if (count <= 0) + return; + + yagpcc::SetPerNodeBatchReq request; + + /* Timestamp */ + *request.mutable_datetime() = current_ts(); + request.set_trace_id(trace_id, GPSC_TRACE_ID_LEN); + + auto *sk = request.mutable_segment_key(); + sk->set_dbid(nodes[0]->dbid); + sk->set_segindex(nodes[0]->segindex); + + for (int i = 0; i < count; i++) + { + GpscNodeSample *node = nodes[i]; + yagpcc::BatchNode *bn = request.add_nodes(); + + bn->set_pid(node->pid); + bn->set_plan_node_id(node->plan_node_id); + bn->set_parent_plan_node_id(node->parent_plan_node_id); + bn->set_node_type(node->node_tag); + bn->set_slice_id(node->slice_id); + bn->set_plan_rows(node->plan_rows); + bn->set_relation_oid(node->relation_oid); + bn->set_ntuples(node->ntuples); + bn->set_tuplecount(node->tuplecount); + bn->set_nloops(node->nloops); + bn->set_startup(node->startup); + bn->set_total(node->total); + bn->set_firsttuple(node->firsttuple); + bn->set_shared_blks_hit(node->shared_blks_hit); + bn->set_shared_blks_read(node->shared_blks_read); + bn->set_node_status(map_node_status(node->node_status)); + bn->set_eof(node->eof); + bn->set_relation_name(node->relation_name); + bn->set_ccnt(node->ccnt); + /* + * executed_at is the snapshot instant, shared by every node in this + * pass. It is stamped per node (not at message level) because the + * receiver aggregates nodes across segments and loses the batch + * grouping; each node needs its own compute time to derive a per-node + * rate. Same value as datetime here since one walk = one instant. + */ + *bn->mutable_executed_at() = request.datetime(); + bn->set_workfile_created(node->workfile_created); + bn->set_workmem_used(node->workmem_used); + bn->set_workmem_wanted(node->workmem_wanted); + /* + * Derived rate fields, computed in signal_handler from the per-node + * rolling state (prev ntuples + prev executed_at). They MUST be + * serialized here too: the receiver keys per invocation trace_id and + * sees each node once, so it cannot re-derive a rate on its side. + */ + bn->set_ntuples_delta(node->ntuples_delta); + bn->set_tuples_per_sec(node->tuples_per_sec); + bn->set_time_since_init_sec(node->time_since_init_sec); + bn->set_stalled(node->stalled); + } + + UDSConnector::report_per_node_batch(request, pne_config); +} + +extern "C" void +gpsc_emit_query_plan(int32_t tmid, int32_t ssid, int32_t ccnt, + const char *plan_doc, int32_t format) +{ + if (plan_doc == nullptr || plan_doc[0] == '\0') + return; + + yagpcc::SetQueryPlanReq request; + + *request.mutable_datetime() = current_ts(); + + auto *qk = request.mutable_query_key(); + qk->set_tmid(tmid); + qk->set_ssid(ssid); + qk->set_ccnt(ccnt); + + request.set_plan_doc(plan_doc); + request.set_format(format); + + UDSConnector::report_query_plan(request, pne_config); +} diff --git a/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.h b/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.h new file mode 100644 index 00000000000..64610353ab1 --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/PlanNodeEmitter.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * PlanNodeEmitter.h + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/PlanNodeEmitter.h + */ + +#ifndef PLAN_NODE_EMITTER_H +#define PLAN_NODE_EMITTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "pg_query_state/qs_types.h" + +extern void gpsc_emit_node_batch(GpscNodeSample **nodes, int count, + const char *trace_id); +extern void gpsc_emit_query_plan(int32_t tmid, int32_t ssid, int32_t ccnt, + const char *plan_doc, int32_t format); +extern void gpsc_qs_sync_config(); + +#ifdef __cplusplus +} +#endif + +#endif /* PLAN_NODE_EMITTER_H */ diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp index 056fa9071a5..6346ba5275d 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.cpp +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.cpp @@ -28,8 +28,8 @@ #include "UDSConnector.h" #include "Config.h" #include "GpscStat.h" -#include "log/LogOps.h" #include "memory/gpdbwrappers.h" +#include "pg_query_state/qs_types.h" #include #include @@ -142,3 +142,154 @@ UDSConnector::report_query(const gpsc::SetQueryReq &req, GpscStat::report_send(total_size); return true; } + +// Extended protocol used by the runtime query-state messages. The high bit of +// the size word tells the receiver that an 8-byte header follows instead of the +// original 4-byte one; the request type word then selects the payload message. +static const uint32_t kExtendedProtocolFlag = 0x80000000u; +static const uint16_t kRequestTypePerNodeBatch = 1; +static const uint16_t kRequestTypeQueryPlan = 2; + +static void inline log_tracing_failure(const yagpcc::SetPerNodeBatchReq &req) +{ + static const char hexchars[] = "0123456789abcdef"; + const unsigned char *trace_id = + reinterpret_cast(req.trace_id().data()); + size_t len = req.trace_id().size(); + char hex[GPSC_TRACE_ID_LEN * 2 + 1]; + + if (len > GPSC_TRACE_ID_LEN) + len = GPSC_TRACE_ID_LEN; + + for (size_t i = 0; i < len; ++i) + { + hex[i * 2] = hexchars[trace_id[i] >> 4]; + hex[i * 2 + 1] = hexchars[trace_id[i] & 0x0f]; + } + hex[len * 2] = '\0'; + + ereport(LOG, + (errmsg("Per-node batch {%s} tracing of %d nodes failed with error %m", + hex, req.nodes_size()))); +} + +static void inline log_tracing_failure(const yagpcc::SetQueryPlanReq &req) +{ + ereport(LOG, + (errmsg("Query {%d-%d-%d} plan document of %zu bytes failed with error %m", + req.query_key().tmid(), req.query_key().ssid(), + req.query_key().ccnt(), req.plan_doc().size()))); +} + +// Sends req behind the 8-byte extended header. Delivery repeats report_query() +// above: a fresh non-blocking socket per message, MSG_DONTWAIT, a nap between +// packets, and a plain drop once the socket refuses to take more. We never wait +// on the reader -- a stalled yagpcc must not slow down the query it observes. +// +// The template exists so that log_tracing_failure() resolves by overload while +// still being called before ~SockGuard() closes the socket and clobbers errno. +template +static bool +report_extended(const Req &req, uint16_t request_type, const Config &config) +{ + sockaddr_un address{}; + address.sun_family = AF_UNIX; + const auto &uds_path = config.uds_path(); + + if (uds_path.size() >= sizeof(address.sun_path)) + { + ereport(WARNING, (errmsg("UDS path is too long for socket buffer"))); + GpscStat::report_error(); + return false; + } + strcpy(address.sun_path, uds_path.c_str()); + + const auto sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) + { + log_tracing_failure(req); + GpscStat::report_error(); + return false; + } + + // Close socket automatically on error path. + struct SockGuard + { + int fd; + ~SockGuard() + { + close(fd); + } + } sock_guard{sockfd}; + + if (fcntl(sockfd, F_SETFL, O_NONBLOCK) == -1) + { + // That's a very important error that should never happen, so make it + // visible to an end-user and admins. + ereport(WARNING, + (errmsg("Unable to create non-blocking socket connection %m"))); + GpscStat::report_error(); + return false; + } + + if (connect(sockfd, reinterpret_cast(&address), + sizeof(address)) == -1) + { + log_tracing_failure(req); + GpscStat::report_bad_connection(); + return false; + } + + const auto data_size = req.ByteSizeLong(); + const auto header_size = sizeof(uint32_t) + 2 * sizeof(uint16_t); + const auto total_size = data_size + header_size; + auto *buf = static_cast(gpdb::palloc(total_size)); + struct BufGuard + { + void *p; + ~BufGuard() + { + gpdb::pfree(p); + } + } buf_guard{buf}; + + *reinterpret_cast(buf) = + static_cast(data_size) | kExtendedProtocolFlag; + *reinterpret_cast(buf + sizeof(uint32_t)) = request_type; + *reinterpret_cast(buf + sizeof(uint32_t) + sizeof(uint16_t)) = 0; + req.SerializeWithCachedSizesToArray(buf + header_size); + + int64_t sent = 0, sent_total = 0; + do + { + sent = send(sockfd, buf + sent_total, total_size - sent_total, + MSG_DONTWAIT); + if (sent > 0) + sent_total += sent; + } while (sent > 0 && size_t(sent_total) != total_size && + (pg_usleep(1000), true)); + + if (sent < 0) + { + log_tracing_failure(req); + GpscStat::report_bad_send(total_size); + return false; + } + + GpscStat::report_send(total_size); + return true; +} + +bool +UDSConnector::report_per_node_batch(const yagpcc::SetPerNodeBatchReq &req, + const Config &config) +{ + return report_extended(req, kRequestTypePerNodeBatch, config); +} + +bool +UDSConnector::report_query_plan(const yagpcc::SetQueryPlanReq &req, + const Config &config) +{ + return report_extended(req, kRequestTypeQueryPlan, config); +} diff --git a/gpcontrib/gp_stats_collector/src/UDSConnector.h b/gpcontrib/gp_stats_collector/src/UDSConnector.h index ac56dd54f44..4258617b190 100644 --- a/gpcontrib/gp_stats_collector/src/UDSConnector.h +++ b/gpcontrib/gp_stats_collector/src/UDSConnector.h @@ -29,6 +29,8 @@ #define UDSCONNECTOR_H #include "protos/gpsc_set_service.pb.h" +#include "protos/yagpcc_plan.pb.h" +#include "protos/yagpcc_set_per_node.pb.h" class Config; @@ -37,6 +39,22 @@ class UDSConnector public: bool static report_query(const gpsc::SetQueryReq &req, const std::string &event, const Config &config); + + // The two calls below use the extended 8-byte header: + // bytes 0-3: payload_size | 0x80000000 (uint32) + // bytes 4-5: request type (uint16) + // bytes 6-7: reserved, zero (uint16) + // bytes 8+: serialized message + // Delivery is the same best-effort push as report_query(): the message is + // dropped when the socket cannot take it. + + // Sends a whole plan-tree snapshot of one backend, request type 1. + bool static report_per_node_batch(const yagpcc::SetPerNodeBatchReq &req, + const Config &config); + + // Sends the coordinator-only deparsed plan document, request type 2. + bool static report_query_plan(const yagpcc::SetQueryPlanReq &req, + const Config &config); }; #endif /* UDSCONNECTOR_H */ diff --git a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c index d295e37b396..686159e0c3d 100644 --- a/gpcontrib/gp_stats_collector/src/gp_stats_collector.c +++ b/gpcontrib/gp_stats_collector/src/gp_stats_collector.c @@ -31,6 +31,7 @@ #include "utils/builtins.h" #include "hook_wrappers.h" +#include "pg_query_state/pg_query_state.h" PG_MODULE_MAGIC; @@ -50,6 +51,14 @@ _PG_init(void) { if (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) hooks_init(); + + /* + * pg_query_state registers its own shared memory, ProcSignal handlers and + * executor hooks. It goes last on purpose: hooks are chained head-first, + * so registering after hooks_init() puts it outside of the collector's + * executor wrappers, which is what it needs to see an untouched QueryDesc. + */ + pg_qs_init(); } void diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/README.md b/gpcontrib/gp_stats_collector/src/pg_query_state/README.md new file mode 100644 index 00000000000..3793d9a4fe7 --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/README.md @@ -0,0 +1,77 @@ +# pg_query_state signal layer + +On-demand inspection of a *running* query: on request, every backend executing +the query (the coordinator and all its QEs) walks its live plan tree and reports +per-node runtime stats, without waiting for the query to finish. + +The code here is derived from [pg_query_state](https://github.com/postgrespro/pg_query_state) +(PostgreSQL License) but the transport and keying differ substantially, so this +note describes the design as implemented in this tree, not the upstream one. + +## Files in this directory + +- `pg_query_state.c` — `_PG_init` wiring, GUCs, executor hooks, the SQL entry + points, the permission gate, and the per-backend `qs_trace_slots` shmem. +- `signal_handler.c` — the two custom ProcSignal handlers, the plan-tree walker, + the per-node delta computation, and the plan-doc builder. +- `qs_types.h` — `GpscNodeSample` (one per-node sample) and the status enums. + +The C++ side that serializes and ships the samples lives one level up in `../` +(`PlanNodeEmitter`, `UDSConnector`, `ProtoUtils`); the wire messages are in +`../../protos/`. The receiver is yagpcc; its side is in `../../../../../yagpcc`. + +## The key: trace_id + +Every collection is keyed by a **trace_id** — 16 raw bytes minted once, on the +coordinator, per `pg_query_state()` call. It is threaded to every participating +backend and stamped into every message. yagpcc groups the snapshots of one poll +by trace_id, and uses it to tell an in-flight poll apart from an overlapping poll +of the same pid. `(tmid, ssid, ccnt)` are display-only. + +## Data flow + +The sequence diagram lives in +[`../../docs/pg_query_state_dataflow.puml`](../../docs/pg_query_state_dataflow.puml) +(render with `plantuml docs/pg_query_state_dataflow.puml`). In short: + +1. **Trigger.** yagpcc mints the trace_id and calls `gpsc.pg_query_state(pid, + trace_id)` on the coordinator (`EXECUTE ON COORDINATOR`). This runs on a fresh + *requestor* backend, not the backend running the observed query. yagpcc + separately calls `gpsc.pg_query_state_backends(pid)` to learn the exact set of + backends to expect — the completeness barrier for the pull below. +2. **Fan-out.** The requestor validates the trace_id and checks the permission + gate, then resolves the observed query's QEs by signalling its coordinator + backend (`BackendInfoPollReason`; `SendCdbComponents` replies over shm_mq). For + each *target* backend it stamps `qs_trace_slots[backendId] = trace_id` and then + signals it with `QueryStatePollReason`: the target QD directly, the target QEs + via a dispatched `cbdb_mpp_query_state(gp_segment_pid[], trace_id)` that runs on + a requestor backend on each segment host. Requestor and target always sit on + the same host and only ever talk by signal. +3. **Collection.** `SendQueryState()` runs in the signalled *target* backend. It + walks the live plan tree, builds one `GpscNodeSample` per node, and pushes the + whole snapshot as a single `SetPerNodeBatchReq` over UDS to the *local* yagpcc + (one connection per backend, not per node). The coordinator additionally builds + the deparsed plan document and sends it as a `SetQueryPlanReq`, rate-limited so + repeated polls of a long query do not resend an unchanged plan. +4. **Storage & pull.** yagpcc stores each batch keyed by trace_id. The master + pulls every segment's batch for that trace, folds in the QD's own nodes, pivots + the flat samples into a per-slice tree, and returns it to the UI. + +## What a node sample carries + +`GpscNodeSample` (see `qs_types.h`) per plan node: identity (`plan_node_id`, +`parent`, `slice_id`, `node_type`, scan `relation_oid`), counters (`ntuples`, +`tuplecount`, `nloops`), timing (`startup`, `total`, `firsttuple`), buffers +(`shared_blks_hit/read`), spill (`workmem_used/wanted`, `workfile_created`), +status (`INITIALIZED`/`EXECUTING`/`FINISHED`, `eof`), and C-side derived rates +(`ntuples_delta`, `tuples_per_sec`, `time_since_init_sec`, `stalled`). The rates +come from a per-node rolling state keyed by `plan_node_id`, reset on executor +start and end. The walk root reports `parent = -1` +(`GPSC_NO_PARENT_PLAN_NODE_ID`); `0` would be ambiguous, since `plan_node_id` +counters start there. + +## Permissions + +The SQL functions are granted to `PUBLIC` so a non-superuser monitoring agent +can run them; access is gated in C: a caller may poll a backend only if it is a +superuser or owns the target query (`GetUserId() == proc->roleId`). diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c new file mode 100644 index 00000000000..1648261526a --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c @@ -0,0 +1,1191 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_query_state.c + * Core of the pg_query_state signal-dispatch layer. + * + * This module provides: + * - Shared-memory setup (shm_toc segment with params, mq, mq_req_id). + * - Custom ProcSignal registrations for three signals: + * QueryStatePollReason -> SendQueryState() + * BackendInfoPollReason -> SendCdbComponents() + * - GUC variables: pg_query_state.enable / enable_timing / enable_buffers. + * - Executor hooks (start/run/finish/end), registered by this module itself, + * that maintain the QueryDescStack and enable instrumentation on the + * top-level query. + * - A requestor-side helper: shm_mq_receive_with_timeout(). + * + * Per-node stats are pushed to the yagpcc UDS sink on demand, when a backend is + * signalled to report its live query state; see signal_handler.c. + * + * Portions derived from pg_query_state + * (https://github.com/postgrespro/pg_query_state), under the PostgreSQL + * License: + * Portions Copyright (c) 2016-2025, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.c + * + *------------------------------------------------------------------------- + */ + +#include "pg_query_state.h" +#include "PlanNodeEmitter.h" + +#include "access/htup_details.h" +#include "access/xact.h" +#include "catalog/pg_type.h" +#include "cdb/cdbdispatchresult.h" +#include "cdb/cdbdisp_query.h" +#include "cdb/cdbexplain.h" +#include "cdb/cdbvars.h" +#include "executor/execParallel.h" +#include "executor/executor.h" +#include "fmgr.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "nodes/nodeFuncs.h" +#include "nodes/print.h" +#include "parser/analyze.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/s_lock.h" +#include "storage/spin.h" +#include "storage/procarray.h" +#include "storage/procsignal.h" +#include "storage/shm_toc.h" +#include "utils/builtins.h" +#include "utils/guc.h" +#include "utils/timestamp.h" +#include "utils/lsyscache.h" +#include "utils/portal.h" +#include "utils/typcache.h" + +/* GUC variables */ +/* Master switch: disabling this suppresses all stat collection. */ +bool pg_qs_enable = true; + +/* Collect timing (wall-clock) data in addition to row counts. */ +bool pg_qs_timing = true; + +/* Collect buffer usage via Instrumentation.bufusage. */ +bool pg_qs_buffers = true; + +/* + * Rolling counter incremented for every QueryDesc pushed onto the stack. + * Used to generate synthetic queryId values for statements lacking one. + */ +static int qs_push_count = 0; + +/* Saved hook pointer for chaining shmem_startup callbacks. */ +static shmem_startup_hook_type prev_shmem_startup_hook = NULL; + +/* Saved hook pointers for chaining the executor callbacks. */ +static ExecutorStart_hook_type prev_ExecutorStart_hook = NULL; +static ExecutorRun_hook_type prev_ExecutorRun_hook = NULL; +static ExecutorFinish_hook_type prev_ExecutorFinish_hook = NULL; +static ExecutorEnd_hook_type prev_ExecutorEnd_hook = NULL; + +/* Whether pg_qs_shmem_startup has completed successfully. */ +static bool module_initialized = false; + +/* + * Monotonically increasing request counter on the requestor side. + * Compared against *mq_req_id in the reply to detect stale responses. + */ +static int reqid = 0; + +/* Shared-memory variables (pointers into the shm_toc segment) */ +/* Table of contents anchoring the whole shared segment. */ +static shm_toc *toc = NULL; + +/* + * Signal parameters written by the requestor and read by the handler. + * Slot 0 in the toc. + */ +pg_qs_params *params = NULL; + +/* + * Raw shared memory queue used to return data from the handler. + * Slot 1 in the toc. + */ +shm_mq *mq = NULL; + +/* + * Shared request-id counter. The requestor increments it before sending a + * signal; the handler echoes it back so the requestor can detect stale + * replies. Slot 2 in the toc. + */ +uint32 *mq_req_id = NULL; + +/* + * Per-backend trace_id slots (toc key 3), indexed by BackendId. The dispatcher + * stamps the target's slot before signalling; the signaled backend reads its + * own slot to key the batch it pushes. See the header for the full rationale. + */ +char (*qs_trace_slots)[GPSC_TRACE_ID_LEN] = NULL; + +/* Global signal-reason handles (set during pg_qs_init) */ +List *QueryDescStack = NIL; + +ProcSignalReason QueryStatePollReason = INVALID_PROCSIGNAL; +ProcSignalReason BackendInfoPollReason = INVALID_PROCSIGNAL; + +/* Forward declarations for module-private helpers */ +static Size pg_qs_shmem_size(void); +static void pg_qs_shmem_startup(void); +static void push_query(QueryDesc *queryDesc); +static void pg_qs_pop_query(void); +static bool filter_query(QueryDesc *queryDesc); +static void pg_qs_executor_start(QueryDesc *queryDesc, int eflags); +static void pg_qs_executor_run(QueryDesc *queryDesc, ScanDirection direction, + uint64 count, bool execute_once); +static void pg_qs_executor_finish(QueryDesc *queryDesc); +static void pg_qs_executor_end(QueryDesc *queryDesc); +static shm_mq_result shm_mq_receive_with_timeout(shm_mq_handle *mqh, Size *nbytesp, + void **datap, int64 timeout); +static List *get_query_backend_info(ArrayType *array); +static shm_mq_result receive_msg_by_parts(shm_mq_handle *mqh, Size *total, + void **datap, int64 timeout, + int *rc, bool nowait); +static PG_QS_RequestResult GetRemoteBackendInfo(PGPROC *proc, List **result); +static void CollectQEQueryState(List *backendInfo, bytea *trace_id); +static void SignalEntryDbBackends(List *backendInfo, bytea *trace_id); +static bool is_querystack_empty(void); +static PG_QS_RequestResult qs_fetch_backend_info(PGPROC *proc, List **backend_info); + +#if PG_VERSION_NUM >= 150000 +static shmem_request_hook_type prev_shmem_request_hook = NULL; +static void pg_qs_shmem_request(void); +#endif + +/* + * pg_qs_shmem_size -- compute the size of the shared memory segment. + * + * The segment holds four objects at fixed toc keys: + * key 0: pg_qs_params + * key 1: message queue of QUEUE_SIZE bytes + * key 2: uint32 request-id counter + * key 3: per-backend trace_id slots, char[GPSC_TRACE_ID_LEN] × (MaxBackends+1) + */ +static Size +pg_qs_shmem_size(void) +{ + shm_toc_estimator e; + Size size; + int nkeys = 4; + + shm_toc_initialize_estimator(&e); + shm_toc_estimate_chunk(&e, sizeof(pg_qs_params)); + shm_toc_estimate_chunk(&e, (Size) QUEUE_SIZE); + shm_toc_estimate_chunk(&e, sizeof(uint32)); + shm_toc_estimate_chunk(&e, (Size) GPSC_TRACE_ID_LEN * (MaxBackends + 1)); + shm_toc_estimate_keys(&e, nkeys); + size = shm_toc_estimate(&e); + return size; +} + +/* + * pg_qs_shmem_startup -- attach to (or initialize) the shared segment. + * + * Called from the shmem_startup_hook chain after shared memory is mapped. + * On first call (found == false) it initialises all sub-structures. + * On subsequent calls it just re-attaches the toc pointers. + */ +static void +pg_qs_shmem_startup(void) +{ + bool found; + Size shmem_size = pg_qs_shmem_size(); + void *shmem; + int num_toc = 0; + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + shmem = ShmemInitStruct("pg_query_state", shmem_size, &found); + if (!found) + { + toc = shm_toc_create(PG_QS_MODULE_KEY, shmem, shmem_size); + + params = shm_toc_allocate(toc, sizeof(pg_qs_params)); + shm_toc_insert(toc, num_toc++, params); + + mq = shm_toc_allocate(toc, QUEUE_SIZE); + shm_toc_insert(toc, num_toc++, mq); + + mq_req_id = shm_toc_allocate(toc, sizeof(uint32)); + shm_toc_insert(toc, num_toc++, mq_req_id); + *mq_req_id = 0; + + qs_trace_slots = shm_toc_allocate(toc, + (Size) GPSC_TRACE_ID_LEN * (MaxBackends + 1)); + shm_toc_insert(toc, num_toc++, qs_trace_slots); + memset(qs_trace_slots, 0, (Size) GPSC_TRACE_ID_LEN * (MaxBackends + 1)); + } + else + { + toc = shm_toc_attach(PG_QS_MODULE_KEY, shmem); + params = shm_toc_lookup(toc, num_toc++, false); + mq = shm_toc_lookup(toc, num_toc++, false); + mq_req_id = shm_toc_lookup(toc, num_toc++, false); + qs_trace_slots = shm_toc_lookup(toc, num_toc++, false); + } + LWLockRelease(AddinShmemInitLock); + + if (prev_shmem_startup_hook) + prev_shmem_startup_hook(); + + module_initialized = true; +} + +#if PG_VERSION_NUM >= 150000 +/* + * pg_qs_shmem_request -- hook called to request shared memory space. + * + * PostgreSQL 15+ separates the request phase from the startup phase. + * This hook is installed only when building against PG15+. + */ +static void +pg_qs_shmem_request(void) +{ + if (prev_shmem_request_hook) + prev_shmem_request_hook(); + + RequestAddinShmemSpace(pg_qs_shmem_size()); +} +#endif + +/* + * pg_qs_init -- initialise the pg_query_state signal infrastructure. + * + * Must be called from _PG_init() while process_shared_preload_libraries_in_progress + * is true. Registers shared memory, custom ProcSignal handlers and GUC + * variables. Safe to call unconditionally for all roles. + */ +void +pg_qs_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + return; + +#if PG_VERSION_NUM >= 150000 + prev_shmem_request_hook = shmem_request_hook; + shmem_request_hook = pg_qs_shmem_request; +#else + RequestAddinShmemSpace(pg_qs_shmem_size()); +#endif + + QueryStatePollReason = RegisterCustomProcSignalHandler(SendQueryState); + BackendInfoPollReason = RegisterCustomProcSignalHandler(SendCdbComponents); + + if (QueryStatePollReason == INVALID_PROCSIGNAL || + BackendInfoPollReason == INVALID_PROCSIGNAL) + { + ereport(WARNING, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), + errmsg("pg_query_state isn't loaded: insufficient custom ProcSignal slots"))); + return; + } + + DefineCustomBoolVariable("pg_query_state.enable", + "Enable module.", + NULL, + &pg_qs_enable, + true, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + DefineCustomBoolVariable("pg_query_state.enable_timing", + "Collect timing data, not just row counts.", + NULL, + &pg_qs_timing, + true, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + DefineCustomBoolVariable("pg_query_state.enable_buffers", + "Collect buffer usage.", + NULL, + &pg_qs_buffers, + true, + PGC_SUSET, + 0, + NULL, NULL, NULL); + + prev_shmem_startup_hook = shmem_startup_hook; + shmem_startup_hook = pg_qs_shmem_startup; + + /* + * Own the executor hooks rather than being called from the collector's + * wrappers: this module has to run outside of whatever else hooks the + * executor, because pg_qs_executor_start() only instruments a query whose + * showstatctx is still unset, and gp_stats_collector allocates one itself + * when gpsc.enable_analyze and gpsc.enable_cdbstats are on. A hook is + * pushed onto the head of the chain, so registering last means running + * first -- see _PG_init() in gp_stats_collector.c. + */ + prev_ExecutorStart_hook = ExecutorStart_hook; + ExecutorStart_hook = pg_qs_executor_start; + prev_ExecutorRun_hook = ExecutorRun_hook; + ExecutorRun_hook = pg_qs_executor_run; + prev_ExecutorFinish_hook = ExecutorFinish_hook; + ExecutorFinish_hook = pg_qs_executor_finish; + prev_ExecutorEnd_hook = ExecutorEnd_hook; + ExecutorEnd_hook = pg_qs_executor_end; + + elog(LOG, "pg_query_state: signal infrastructure initialised"); +} + +/* Executor lifecycle hooks */ +/* + * pg_qs_executor_start -- called at the start of executor execution. + * + * Enables instrumentation on the QueryDesc when: + * - pg_query_state is enabled + * - this is not an EXPLAIN-only execution + * - we are on a QD or QE role + * - there is no outer query already on the stack (top-level only) + * - the query passes the filter + * - no showstatctx is already attached + * + * Also assigns a synthetic queryId when the planner left it as zero. + * + * Parameters: + * queryDesc -- the QueryDesc being started + * eflags -- executor flags (EXEC_FLAG_EXPLAIN_ONLY etc.) + */ +static void +pg_qs_executor_start(QueryDesc *queryDesc, int eflags) +{ + instr_time starttime; + + if (pg_qs_enable + && ((eflags & EXEC_FLAG_EXPLAIN_ONLY) == 0) + && (Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_EXECUTE) + && is_querystack_empty() + && filter_query(queryDesc) + && queryDesc->showstatctx == NULL) + { + queryDesc->instrument_options |= INSTRUMENT_CDB; + queryDesc->instrument_options |= INSTRUMENT_ROWS; + if (pg_qs_timing) + queryDesc->instrument_options |= INSTRUMENT_TIMER; + if (pg_qs_buffers) + queryDesc->instrument_options |= INSTRUMENT_BUFFERS; + + INSTR_TIME_SET_CURRENT(starttime); + + /* + * cdbexplain_showExecStatsBegin() aggregates QE stats on the QD and + * asserts Gp_role != GP_ROLE_EXECUTE, so it must run on the dispatcher + * only. QE backends still get instrument_options above, which is all + * the per-node walker reads. + */ + if (Gp_role == GP_ROLE_DISPATCH) + queryDesc->showstatctx = + cdbexplain_showExecStatsBegin(queryDesc, starttime); + queryDesc->totaltime = InstrAlloc(1, INSTRUMENT_ALL, false); + + gpsc_reset_node_roll_state(); + } + + if (queryDesc->plannedstmt->queryId == 0) + queryDesc->plannedstmt->queryId = + ((uint64) gp_command_count << 32) + qs_push_count; + + if (prev_ExecutorStart_hook) + prev_ExecutorStart_hook(queryDesc, eflags); + else + standard_ExecutorStart(queryDesc, eflags); +} + +/* + * pg_qs_executor_run -- called when the executor begins fetching tuples. + * + * Keeps the QueryDesc on the stack for as long as tuples are being fetched, so + * that a poll arriving mid-run finds it. + */ +static void +pg_qs_executor_run(QueryDesc *queryDesc, ScanDirection direction, + uint64 count, bool execute_once) +{ + push_query(queryDesc); + PG_TRY(); + { + if (prev_ExecutorRun_hook) + prev_ExecutorRun_hook(queryDesc, direction, count, execute_once); + else + standard_ExecutorRun(queryDesc, direction, count, execute_once); + } + PG_FINALLY(); + { + pg_qs_pop_query(); + } + PG_END_TRY(); +} + +/* + * pg_qs_executor_finish -- called after all tuples have been fetched. + * + * Same push/pop as the run phase: the query stays visible to signal handlers + * while after-triggers and the like are still running. + */ +static void +pg_qs_executor_finish(QueryDesc *queryDesc) +{ + push_query(queryDesc); + PG_TRY(); + { + if (prev_ExecutorFinish_hook) + prev_ExecutorFinish_hook(queryDesc); + else + standard_ExecutorFinish(queryDesc); + } + PG_FINALLY(); + { + pg_qs_pop_query(); + } + PG_END_TRY(); +} + +/* + * pg_qs_executor_end -- called when executor resources are released. + * + * Drops the per-node rolling state so the next query on this backend starts its + * delta accounting clean. It does not collect or push anything: a finish is not + * a signalled collection and carries no trace_id to key a batch under. + */ +static void +pg_qs_executor_end(QueryDesc *queryDesc) +{ + if (queryDesc && pg_qs_enable) + gpsc_reset_node_roll_state(); + + if (prev_ExecutorEnd_hook) + prev_ExecutorEnd_hook(queryDesc); + else + standard_ExecutorEnd(queryDesc); +} + +static void +push_query(QueryDesc *queryDesc) +{ + qs_push_count++; + QueryDescStack = lcons(queryDesc, QueryDescStack); +} + +static void +pg_qs_pop_query(void) +{ + QueryDescStack = list_delete_first(QueryDescStack); +} + +static bool +is_querystack_empty(void) +{ + return list_length(QueryDescStack) == 0; +} + +QueryDesc * +get_toppest_query(void) +{ + return (QueryDescStack == NIL) ? NULL : (QueryDesc *) llast(QueryDescStack); +} + +/* + * filter_query -- decide whether to instrument a given QueryDesc. + * + * Returns false for cursor queries with non-default cursor options, and for + * utility statements. Returns true for SELECT, INSERT, UPDATE, DELETE. + */ +static bool +filter_query(QueryDesc *queryDesc) +{ + Portal portal; + + if (queryDesc == NULL) + return false; + + if (queryDesc->extended_query && queryDesc->portal_name) + { + portal = GetPortalByName(queryDesc->portal_name); + if (!PointerIsValid(portal) || portal->cursorOptions != CURSOR_OPT_NO_SCROLL) + return false; + } + + return (queryDesc->operation == CMD_SELECT || + queryDesc->operation == CMD_DELETE || + queryDesc->operation == CMD_INSERT || + queryDesc->operation == CMD_UPDATE); +} + +/* + * LockShmem -- acquire an exclusive user-lock keyed by (PG_QS_MODULE_KEY, key). + * + * Used to serialise access to the shared mq between concurrent requestors + * and between requestor and handler. + */ +static void +LockShmem(LOCKTAG *tag, uint32 key) +{ + LockAcquireResult result; + + tag->locktag_field1 = PG_QS_MODULE_KEY; + tag->locktag_field2 = key; + tag->locktag_field3 = 0; + tag->locktag_field4 = 0; + tag->locktag_type = LOCKTAG_USERLOCK; + tag->locktag_lockmethodid = USER_LOCKMETHOD; + + result = LockAcquire(tag, ExclusiveLock, false, false); + Assert(result == LOCKACQUIRE_OK); +} + +/* + * UnlockShmem -- release the exclusive user-lock acquired by LockShmem. + */ +static void +UnlockShmem(LOCKTAG *tag) +{ + LockRelease(tag, ExclusiveLock, false); +} + +/* + * GetRemoteBackendInfo -- obtain the list of (segid, pid) pairs from QD. + * + * Sends BackendInfoPollReason to proc and waits for the reply. On success, + * *result is populated with gp_segment_pid entries (palloc'd). + * + * Returns the PG_QS_RequestResult code from the reply. + */ +static PG_QS_RequestResult +GetRemoteBackendInfo(PGPROC *proc, List **result) +{ + int sig_result; + shm_mq_handle *mqh; + shm_mq_result mq_receive_result; + Size msg_len; + backend_info *msg; + LOCKTAG tag; + int i; + + LockShmem(&tag, PG_QS_SND_KEY); + params->reason = BackendInfoPollReason; + mq = shm_mq_create(mq, QUEUE_SIZE); + shm_mq_set_sender(mq, proc); + shm_mq_set_receiver(mq, MyProc); + *mq_req_id = reqid; + UnlockShmem(&tag); + + sig_result = SendProcSignal(proc->pid, BackendInfoPollReason, + proc->backendId); + if (sig_result == -1) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not send BackendInfoPollReason signal"))); + + mqh = shm_mq_attach(mq, NULL, NULL); + mq_receive_result = shm_mq_receive_with_timeout(mqh, &msg_len, + (void **) &msg, + MAX_RCV_TIMEOUT); + + if (mq_receive_result != SHM_MQ_SUCCESS || msg == NULL || + msg->reqid != (uint32) reqid) + { + shm_mq_detach(mqh); + ereport(WARNING, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("GetRemoteBackendInfo: message not received"))); + return QUERY_NOT_RUNNING; + } + + if (msg->result_code != QS_RETURNED) + { + PG_QS_RequestResult result_code = msg->result_code; + shm_mq_detach(mqh); + return result_code; + } + + /* Validate the reply payload length against the reported backend count. */ + { + int expected_len = BASE_SIZEOF_GP_BACKEND_INFO + + msg->number * sizeof(gp_segment_pid); + if ((int) msg_len != expected_len) + { + shm_mq_detach(mqh); + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("GetRemoteBackendInfo: unexpected message length"))); + } + } + + for (i = 0; i < msg->number; i++) + { + gp_segment_pid *segpid = palloc(sizeof(gp_segment_pid)); + *segpid = msg->pids[i]; + *result = lcons(segpid, *result); + } + + shm_mq_detach(mqh); + return QS_RETURNED; +} + +/* + * CollectQEQueryState -- fan-out query-state signals to the segment QEs. + * + * Dispatches a cbdb_mpp_query_state() call to each segment listed in + * backendInfo. Results are returned as raw CdbPgResults. + * + * GPSC_SEGID_ENTRY_DB entries are left out: the dispatch reaches primary + * segments only, and there the receiving cbdb_mpp_query_state() matches + * entries against its own GpIdentity.segindex, which is never negative. + * SignalEntryDbBackends() handles those. + */ +static void +CollectQEQueryState(List *backendInfo, bytea *trace_id) +{ + ListCell *lc; + StringInfoData params_buf; + char *sql; + char trace_id_hex[2 * GPSC_TRACE_ID_LEN + 1]; + int nsegments = 0; + + if (list_length(backendInfo) == 0) + return; + + initStringInfo(¶ms_buf); + + foreach(lc, backendInfo) + { + gp_segment_pid *segpid = (gp_segment_pid *) lfirst(lc); + + if (segpid->segid < 0) + continue; + + if (nsegments++ > 0) + appendStringInfoChar(¶ms_buf, ','); + appendStringInfo(¶ms_buf, "'(%d,%d)'", segpid->segid, segpid->pid); + } + + if (nsegments == 0) + { + pfree(params_buf.data); + return; + } + + hex_encode(VARDATA_ANY(trace_id), GPSC_TRACE_ID_LEN, trace_id_hex); + trace_id_hex[2 * GPSC_TRACE_ID_LEN] = '\0'; + sql = psprintf("SELECT gpsc.cbdb_mpp_query_state((ARRAY[%s])::gpsc.gp_segment_pid[], '\\x%s'::bytea)", + params_buf.data, trace_id_hex); + + CdbDispatchCommand(sql, DF_NONE, NULL); + pfree(params_buf.data); + pfree(sql); +} + +/* + * SignalEntryDbBackends -- poll the entry-db QEs listed in backendInfo. + * + * An entry-db reader runs the coordinator-side slice of a distributed query and + * so holds the only instrumentation for it, but it lives in the coordinator's + * own postmaster and no dispatch reaches it. Since it is a local backend, the + * QD signals it the same way it signals itself. + */ +static void +SignalEntryDbBackends(List *backendInfo, bytea *trace_id) +{ + ListCell *lc; + + foreach(lc, backendInfo) + { + gp_segment_pid *segpid = (gp_segment_pid *) lfirst(lc); + PGPROC *proc; + + if (segpid->segid != GPSC_SEGID_ENTRY_DB) + continue; + + proc = BackendPidGetProc(segpid->pid); + if (!proc || proc->backendId == InvalidBackendId) + continue; + + memcpy(qs_trace_slots[proc->backendId], VARDATA_ANY(trace_id), + GPSC_TRACE_ID_LEN); + if (SendProcSignal(proc->pid, QueryStatePollReason, + proc->backendId) == -1) + elog(DEBUG1, "pg_query_state: failed to signal entry-db backend pid=%d", + segpid->pid); + } +} + +/* + * shm_mq_receive_with_timeout -- receive from mqh, blocking up to `timeout` ms. + * + * Calls receive_msg_by_parts() in a loop, sleeping on the latch between + * retries. Returns SHM_MQ_SUCCESS, SHM_MQ_DETACHED, or SHM_MQ_WOULD_BLOCK + * (the last meaning the timeout expired). + * + * On success, *nbytesp is set to the message length and *datap to a palloc'd + * buffer containing the message. + */ +static shm_mq_result +shm_mq_receive_with_timeout(shm_mq_handle *mqh, + Size *nbytesp, + void **datap, + int64 timeout) +{ + int rc = 0; + int64 delay = timeout; + instr_time start_time; + instr_time cur_time; + + INSTR_TIME_SET_CURRENT(start_time); + + for (;;) + { + shm_mq_result result; + + result = receive_msg_by_parts(mqh, nbytesp, datap, timeout, &rc, true); + if (result != SHM_MQ_WOULD_BLOCK) + return result; + + if (rc & WL_TIMEOUT || delay <= 0) + return SHM_MQ_WOULD_BLOCK; + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT, + delay, PG_WAIT_EXTENSION); + + INSTR_TIME_SET_CURRENT(cur_time); + INSTR_TIME_SUBTRACT(cur_time, start_time); + delay = timeout - (int64) INSTR_TIME_GET_MILLISEC(cur_time); + if (delay <= 0) + return SHM_MQ_WOULD_BLOCK; + + CHECK_FOR_INTERRUPTS(); + ResetLatch(MyLatch); + } +} + +/* + * receive_msg_by_parts -- reassemble a multi-chunk message from mqh. + * + * The wire protocol prefixes each message with its total byte count (a Size), + * followed by one or more chunks of up to MSG_MAX_SIZE bytes. This function + * reads the prefix, allocates a buffer, and loops until all chunks arrive. + * + * Parameters: + * mqh -- attached message-queue handle + * total -- out: total bytes received + * datap -- out: palloc'd buffer with reassembled message + * timeout -- caller's deadline in ms (used only for PART_RCV_DELAY retries) + * rc -- out: WaitLatch flags (set to WL_TIMEOUT if we give up) + * nowait -- passed through to shm_mq_receive + */ +static shm_mq_result +receive_msg_by_parts(shm_mq_handle *mqh, Size *total, void **datap, + int64 timeout, int *rc, bool nowait) +{ + shm_mq_result mq_receive_result; + shm_mq_msg *buff; + int offset; + Size *expected; + Size expected_data; + Size len; + + /* Read the length prefix. */ + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &expected, nowait); + if (mq_receive_result != SHM_MQ_SUCCESS) + return mq_receive_result; + Assert(len == sizeof(Size)); + + expected_data = *expected; + Assert(expected_data < UINT32_MAX); + *datap = palloc0(expected_data); + + /* Reassemble chunks until we have expected_data bytes. */ + for (offset = 0; offset < (int) expected_data; ) + { + int64 delay = timeout; + + for (;;) + { + mq_receive_result = shm_mq_receive(mqh, &len, (void **) &buff, + nowait); + if (mq_receive_result != SHM_MQ_SUCCESS) + { + if (nowait && mq_receive_result == SHM_MQ_WOULD_BLOCK) + { + if (delay > 0) + { + pg_usleep(PART_RCV_DELAY * 1000); + delay -= PART_RCV_DELAY; + continue; + } + if (rc) + *rc |= WL_TIMEOUT; + } + return mq_receive_result; + } + break; + } + memcpy((char *) *datap + offset, buff, len); + offset += len; + } + + *total = offset; + return mq_receive_result; +} + +/* + * qs_fetch_backend_info -- serialise a backend-info request and collect the + * (segid, pid) list for the query running on `proc`. + * + * Holds PG_QS_RCV_KEY across the request so concurrent requestors do not clobber + * the shared mq, releasing it on both the success and error paths. + */ +static PG_QS_RequestResult +qs_fetch_backend_info(PGPROC *proc, List **backend_info) +{ + LOCKTAG tag; + PG_QS_RequestResult result; + + LockShmem(&tag, PG_QS_RCV_KEY); + PG_TRY(); + { + reqid = *mq_req_id + 1; + result = GetRemoteBackendInfo(proc, backend_info); + UnlockShmem(&tag); + } + PG_CATCH(); + { + UnlockShmem(&tag); + PG_RE_THROW(); + } + PG_END_TRY(); + + return result; +} + +/* SQL callable functions */ +/* + * pg_query_state -- entry point for the pg_query_state() SQL function. + * + * Obtains the user-id and segment-backend list from the target backend, + * then fans out cbdb_mpp_query_state() to each QE. + */ +PG_FUNCTION_INFO_V1(pg_query_state); +Datum +pg_query_state(PG_FUNCTION_ARGS) +{ + pid_t pid = PG_GETARG_INT32(0); + bytea *trace_id = PG_GETARG_BYTEA_P(1); + PGPROC *proc; + PG_QS_RequestResult result; + List *backend_info = NIL; + + if (VARSIZE_ANY_EXHDR(trace_id) != GPSC_TRACE_ID_LEN) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid size of trace_id: %zu, expected %d", + VARSIZE_ANY_EXHDR(trace_id), GPSC_TRACE_ID_LEN))); + + if (pid == MyProcPid) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot extract state of current process"))); + + if (!module_initialized) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_query_state must be loaded via shared_preload_libraries"))); + + proc = BackendPidGetProc(pid); + if (!proc || proc->backendId == InvalidBackendId || + proc->databaseId == InvalidOid || proc->roleId == InvalidOid) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("backend with pid=%d not found", pid))); + + if (!(superuser() || GetUserId() == proc->roleId)) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied"))); + } + + result = qs_fetch_backend_info(proc, &backend_info); + + switch (result) + { + case QUERY_NOT_RUNNING: + elog(DEBUG1, "pg_query_state: pid=%d is not running a query", pid); + break; + + case STAT_DISABLED: + elog(DEBUG1, "pg_query_state: stats collection disabled"); + break; + + case WRONG_ROLE: + /* + * Not the QD, so there is no participant list to fan out to and no + * point signalling: a QE polled directly would report a single + * slice that no collection is waiting for. Stay quiet here -- the + * caller-facing complaint belongs to pg_query_state_backends(), + * which errors out on the same result code. + */ + elog(DEBUG1, "pg_query_state: pid=%d is a query executor, not the QD", pid); + break; + + case QS_RETURNED: + /* + * Signal all segment QEs to push their plan-node stats via UDS, + * carrying the trace_id so every backend's batch lands under the one + * key this pg_query_state() invocation owns. + */ + CollectQEQueryState(backend_info, trace_id); + SignalEntryDbBackends(backend_info, trace_id); + + /* + * Signal the QD backend itself so it pushes coordinator-side plan + * nodes and the plan-doc. SendQueryState() emits directly via UDS. + * Stamp the target's own trace slot before signalling, so its batch + * lands under this collection's key. + */ + memcpy(qs_trace_slots[proc->backendId], VARDATA_ANY(trace_id), + GPSC_TRACE_ID_LEN); + SendProcSignal(proc->pid, QueryStatePollReason, proc->backendId); + break; + } + + PG_RETURN_VOID(); +} + +/* + * pg_query_state_backends -- list the QE backends participating in the query + * running on backend `pid`. + * + * Returns a set of (segid, pid) rows obtained from the coordinator via + * GetRemoteBackendInfo (the same list the poll path fans out to). A consumer + * can use the row count as the expected number of backends that will report. + * + * Uses the materialize SRF mode: the whole list is built into a tuplestore in + * one call. Returns an empty set when the target query is not running. + */ +PG_FUNCTION_INFO_V1(pg_query_state_backends); +Datum +pg_query_state_backends(PG_FUNCTION_ARGS) +{ + pid_t pid = PG_GETARG_INT32(0); + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + PGPROC *proc; + List *backend_info = NIL; + PG_QS_RequestResult result; + ListCell *lc; + + InitMaterializedSRF(fcinfo, 0); + tupdesc = rsinfo->setDesc; + tupstore = rsinfo->setResult; + + if (pid == MyProcPid) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cannot extract state of current process"))); + + proc = BackendPidGetProc(pid); + if (!proc || proc->backendId == InvalidBackendId || + proc->databaseId == InvalidOid || proc->roleId == InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("backend with pid=%d not found", pid))); + + if (!module_initialized) + { + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("pg_query_state must be loaded via shared_preload_libraries"))); + } + + if (!(superuser() || GetUserId() == proc->roleId)) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied"))); + } + + result = qs_fetch_backend_info(proc, &backend_info); + + /* + * Not running / disabled are ordinary outcomes of polling a pid that has + * just finished: return an empty set rather than erroring. A wrong-role + * target is different -- the backend is alive and will never answer, which + * a caller must be able to tell apart from a finished query, so that one + * does error out. + */ + switch (result) + { + case QUERY_NOT_RUNNING: + elog(DEBUG1, "pg_query_state_backends: pid=%d is not running a query", pid); + return (Datum) 0; + + case STAT_DISABLED: + elog(DEBUG1, "pg_query_state_backends: stats collection disabled"); + return (Datum) 0; + + case WRONG_ROLE: + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("backend with pid=%d is a query executor, " + "not the session's coordinator backend", pid))); + break; + + case QS_RETURNED: + break; + } + + foreach(lc, backend_info) + { + gp_segment_pid *segpid = (gp_segment_pid *) lfirst(lc); + Datum values[2]; + bool nulls[2] = {false, false}; + + values[0] = Int32GetDatum(segpid->segid); + values[1] = Int32GetDatum(segpid->pid); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + /* + * QD-only query (INSERT ... VALUES, catalog reads, and other coordinator- + * local plans): no QE gang ran, so backend_info is empty even though the + * coordinator is executing and will push its own per-node batch. Report the + * coordinator itself so the caller does not mistake an empty QE list for a + * finished query and drop the QD's batch. + */ + if (list_length(backend_info) == 0) + { + Datum values[2]; + bool nulls[2] = {false, false}; + + values[0] = Int32GetDatum(GPSC_SEGID_QD); + values[1] = Int32GetDatum(proc->pid); + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + return (Datum) 0; +} + +/* + * cbdb_mpp_query_state -- QE-side entry point dispatched by CollectQEQueryState. + * + * Receives an array of gp_segment_pid, filters those belonging to this + * segment, and fires QueryStatePollReason at each matching backend. + */ +PG_FUNCTION_INFO_V1(cbdb_mpp_query_state); +Datum +cbdb_mpp_query_state(PG_FUNCTION_ARGS) +{ + ListCell *iter; + List *alive_procs = get_query_backend_info(PG_GETARG_ARRAYTYPE_P(0)); + bytea *trace_id = PG_GETARG_BYTEA_P(1); + + if (VARSIZE_ANY_EXHDR(trace_id) != GPSC_TRACE_ID_LEN) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid size of trace_id: %zu, expected %d", + VARSIZE_ANY_EXHDR(trace_id), GPSC_TRACE_ID_LEN))); + + if (alive_procs == NIL) + PG_RETURN_NULL(); + + foreach(iter, alive_procs) + { + PGPROC *proc = (PGPROC *) lfirst(iter); + int sig_result; + + if (!proc || proc->backendId == InvalidBackendId) + continue; + + /* Stamp the target's own trace slot before signalling it. */ + memcpy(qs_trace_slots[proc->backendId], VARDATA_ANY(trace_id), + GPSC_TRACE_ID_LEN); + + sig_result = SendProcSignal(proc->pid, QueryStatePollReason, + proc->backendId); + if (sig_result == -1) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cbdb_mpp_query_state: failed to send signal to pid %d", + proc->pid))); + } + PG_RETURN_VOID(); +} + +/* + * get_query_backend_info -- convert a gp_segment_pid[] SQL array to a list + * of PGPROC pointers for backends running on this segment. + * + * Skips entries for other segments and entries whose backend has exited. + */ +static List * +get_query_backend_info(ArrayType *array) +{ + int16 typlen; + bool typbyval; + char typalign; + Oid element_type = ARR_ELEMTYPE(array); + Datum *data; + bool *nulls; + int nitems; + int len; + List *alive_procs = NIL; + + get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); + deconstruct_array(array, element_type, typlen, typbyval, typalign, + &data, &nulls, &nitems); + + len = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + + for (int i = 0; i < len; i++) + { + if (nulls[i]) + continue; + + HeapTupleHeader td = DatumGetHeapTupleHeader(data[i]); + TupleDesc tupdesc; + HeapTupleData tmptup; + int32 pid; + int32 segid; + bool segid_isnull = false; + bool pid_isnull = false; + PGPROC *proc; + + tupdesc = lookup_rowtype_tupdesc_copy( + HeapTupleHeaderGetTypeId(td), HeapTupleHeaderGetTypMod(td)); + tmptup.t_len = HeapTupleHeaderGetDatumLength(td); + tmptup.t_data = td; + + segid = DatumGetInt32(heap_getattr(&tmptup, 1, tupdesc, &segid_isnull)); + pid = DatumGetInt32(heap_getattr(&tmptup, 2, tupdesc, &pid_isnull)); + FreeTupleDesc(tupdesc); + + if (segid_isnull || pid_isnull || segid != GpIdentity.segindex) + continue; + + proc = BackendPidGetProc(pid); + if (!proc) + continue; + + alive_procs = lappend(alive_procs, proc); + } + return alive_procs; +} diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h new file mode 100644 index 00000000000..4f532631958 --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h @@ -0,0 +1,228 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_query_state.h + * Public API for the pg_query_state signal-dispatch layer. + * + * This header is included by the C extension entry point (gp_stats_collector.c), + * which only has to call pg_qs_init(); everything else the module needs it + * registers itself. Keep it C-compatible: no C++ types, wrapped in extern "C". + * + * Portions derived from pg_query_state + * (https://github.com/postgrespro/pg_query_state), under the PostgreSQL + * License: + * Portions Copyright (c) 2016-2025, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/pg_query_state.h + * + *------------------------------------------------------------------------- + */ +#ifndef __PG_QUERY_STATE_H__ +#define __PG_QUERY_STATE_H__ +#ifdef __cplusplus +extern "C" { +#endif + +#include "postgres.h" + +#include "commands/explain.h" +#include "nodes/pg_list.h" +#include "storage/procarray.h" +#include "storage/shm_mq.h" +#include "cdb/cdbdispatchresult.h" +#include "qs_types.h" + +/* Shared memory queue capacity for passing query-state messages. */ +#define QUEUE_SIZE (64 * 1024) + +/* Maximum single chunk size when splitting a message across shm_mq sends. */ +#define MSG_MAX_SIZE (4 * 1024) + +/* Delay between shm_mq send retries, in microseconds (100 ms). */ +#define WRITING_DELAY (100 * 1000) + +/* Maximum number of send retries before giving up. */ +#define NUM_OF_ATTEMPTS 6 + +/* Bitmask flags for caller-side warnings embedded in shm_mq_msg.warnings. */ +#define TIMING_OFF_WARNING 1 +#define BUFFERS_OFF_WARNING 2 + +/* Unique key that identifies our shm_toc segment. */ +#define PG_QS_MODULE_KEY 0xCA94B108 + +/* Table-of-contents slot indices within the shm_toc segment. */ +#define PG_QS_RCV_KEY 0 +#define PG_QS_SND_KEY 1 + +/* + * Timeouts for shm_mq operations. + * The receive timeout must exceed the send timeout so that waiting workers + * always give up before the polling process stops listening. + */ +#define MAX_RCV_TIMEOUT 2000 /* ms */ +#define MAX_SND_TIMEOUT 1000 /* ms */ + +/* + * Sleep between partial-receive retries (SHM_MQ_WOULD_BLOCK case). + * Must be less than MAX_RCV_TIMEOUT. + */ +#define PART_RCV_DELAY 100 /* ms */ + +/* + * Minimum interval between coordinator plan-doc pushes for the same query. + * SendQueryState() re-sends the ExplainPrintPlan document only after this + * interval elapses, so repeated polls of a long-running query do not resend + * the (unchanging) plan on every signal. + */ +#define PLAN_DOC_RESEND_INTERVAL_MS (2 * 60 * 1000) + +/* + * Status codes returned by the signal handler to describe the state of the + * queried backend. + */ +typedef enum +{ + QUERY_NOT_RUNNING, /* backend is idle or has no active QueryDesc */ + STAT_DISABLED, /* pg_query_state.enable = false */ + QS_RETURNED, /* handler successfully collected and sent stats */ + WRONG_ROLE /* target is a QE, not the QD: only GP_ROLE_DISPATCH + * knows the participant list, so no other backend can + * answer BackendInfoPollReason */ +} PG_QS_RequestResult; + +/* + * Wire format for a query-state reply message transmitted through shm_mq. + * The variable-length `stack` field carries sequentially laid out text frames, + * one per stack depth. + */ +typedef struct +{ + int reqid; + int length; /* total message size including flexible array */ + PGPROC *proc; + PG_QS_RequestResult result_code; + int warnings; /* bitmask of TIMING_OFF_WARNING / BUFFERS_OFF_WARNING */ + int stack_depth; + char stack[FLEXIBLE_ARRAY_MEMBER]; +} shm_mq_msg; + +#define BASE_SIZEOF_SHM_MQ_MSG (offsetof(shm_mq_msg, stack_depth)) + +/* + * Compact identifier for a backend running on a specific segment. + */ +typedef struct +{ + int32 segid; + int32 pid; +} gp_segment_pid; + +/* + * Wire format for the backend-info (CDB segment PIDs) reply. + */ +typedef struct +{ + int reqid; + int length; + PGPROC *proc; + PG_QS_RequestResult result_code; + int number; + gp_segment_pid pids[FLEXIBLE_ARRAY_MEMBER]; +} backend_info; + +#define BASE_SIZEOF_GP_BACKEND_INFO (offsetof(backend_info, pids)) + +/* + * Parameters passed through shared memory from the requestor to the signal + * handler, controlling what the handler should collect and how. + */ +typedef struct +{ + ProcSignalReason reason; + int reqid; +} pg_qs_params; + +/* + * Context threaded through the plan-tree walker. + * per_node_stats accumulates one GpscNodeSample per visited node. + */ +typedef struct QsWalkerContext +{ + List *per_node_stats; + int32_t parent_plan_node_id; + int32_t slice_id; /* slice owning the node being visited */ + bool finalize; /* true only in pg_qs_executor end */ + TimestampTz ts_now; + int32_t tmid; +} QsWalkerContext; + +/* + * Result code for the chunked shm_mq send helper. + */ +typedef enum +{ + MSG_BY_PARTS_SUCCEEDED, + MSG_BY_PARTS_FAILED +} msg_by_parts_result; + +extern bool pg_qs_enable; +extern bool pg_qs_timing; +extern bool pg_qs_buffers; +extern List *QueryDescStack; +extern pg_qs_params *params; +extern shm_mq *mq; +extern uint32 *mq_req_id; + +/* + * Per-backend trace_id slots, indexed by BackendId (1..MaxBackends; slot 0 for + * InvalidBackendId is unused). The single shared `params` cannot carry the + * trace across an asynchronous ProcSignal: two concurrent collections would + * clobber it and a signaled backend would stamp its batch with the wrong + * trace. The dispatcher writes qs_trace_slots[target->backendId] before + * signalling; the signaled backend reads qs_trace_slots[MyBackendId]. The slot + * is keyed by backend, not by collection, so two overlapping collections of the + * same backend still share one slot -- the caller must not poll one pid twice + * concurrently. + */ +extern char (*qs_trace_slots)[GPSC_TRACE_ID_LEN]; + +extern ProcSignalReason QueryStatePollReason; +extern ProcSignalReason BackendInfoPollReason; + +/* + * pg_qs_init -- register shared memory, custom signals, GUC variables and the + * executor hooks. Must be called from _PG_init() during + * shared_preload_libraries processing. + */ +extern void pg_qs_init(void); + +/* Custom signal handlers registered with RegisterCustomProcSignalHandler. */ +extern void SendQueryState(void); +extern void SendCdbComponents(void); + +typedef void (*qs_planstate_walker_callback)(PlanState *, QsWalkerContext *); +extern QueryDesc *get_toppest_query(void); +extern void gpsc_reset_node_roll_state(void); + +#ifdef __cplusplus +} +#endif +#endif /* __PG_QUERY_STATE_H__ */ diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h b/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h new file mode 100644 index 00000000000..79312f09acf --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h @@ -0,0 +1,112 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * qs_types.h + * Per-node sample type collected by the pg_query_state plan-tree walker. + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/qs_types.h + * + *------------------------------------------------------------------------- + */ +#ifndef QS_TYPES_H +#define QS_TYPES_H + +#include +#include + +#define GPSC_TRACE_ID_LEN 16 +#define MAX_RELNAME_LEN 64 + +/* + * parent_plan_node_id of a root node. Not 0: plan_node_id counters start at 0 + * (setrefs.c, and GPORCA's GetNextPlanId), so 0 is always a real node. + */ +#define GPSC_NO_PARENT_PLAN_NODE_ID (-1) + +/* + * Negative segids reported in per-node samples and in the participant list. + * Real segments report GpIdentity.segindex, which is >= 0. On the coordinator + * host it is -1 for the QD and for the entry-db QE alike, so the entry-db is + * re-stamped: consumers key their dedup and their barrier on this value, and + * two backends sharing it means one of them is silently dropped. + */ +#define GPSC_SEGID_QD (-1) +#define GPSC_SEGID_ENTRY_DB (-2) + +/* + * Execution phase of a single plan node as observed at signal time. + */ +typedef enum QsNodeStatus +{ + QS_NODE_STATUS_UNSPECIFIED = 0, + QS_NODE_STATUS_INITIALIZED = 1, /* instrumentation allocated but not yet started */ + QS_NODE_STATUS_EXECUTING = 2, /* currently inside a tuple-fetch call */ + QS_NODE_STATUS_FINISHED = 3 /* at least one full loop completed */ +} QsNodeStatus; + +typedef struct GpscNodeSample +{ + int32_t tmid; /* transaction/time id (gp_gettmid) */ + int32_t ssid; /* gp_session_id */ + int32_t ccnt; /* gp_command_count */ + int32_t plan_node_id; /* Plan.plan_node_id */ + int32_t parent_plan_node_id; /* parent's plan_node_id, or + * GPSC_NO_PARENT_PLAN_NODE_ID at the root */ + int32_t node_tag; /* nodeTag(plan) */ + int32_t slice_id; /* currentSliceId */ + int32_t segindex; /* GpIdentity.segindex */ + int32_t pid; /* MyProcPid of the sampled backend */ + int32_t dbid; /* GpIdentity.dbid */ + int32_t relation_oid; /* OID of scanned relation, or 0 */ + double plan_rows; /* optimizer row estimate */ + double ntuples; /* Instrumentation.ntuples */ + double tuplecount; /* Instrumentation.tuplecount (in-progress loop) */ + double nloops; /* Instrumentation.nloops */ + double startup; /* Instrumentation.startup (seconds) */ + double total; /* Instrumentation.total (seconds) */ + double firsttuple; /* Instrumentation.firsttuple (seconds) */ + uint64_t shared_blks_hit; + uint64_t shared_blks_read; + QsNodeStatus node_status; + bool eof; /* Instrumentation.eof: node exhausted for + * the current cycle (last fetch returned no + * tuple). Lets consumers tell a finished + * node from one still actively producing. */ + /* + * Spill, from the GP-specific Instrumentation fields. Reliable once the node + * is finalized; a mid-run snapshot is a lower bound (Sort/HashJoin populate + * these only at eager-free / explain-end). + */ + bool workfile_created; /* Instrumentation.workfileCreated */ + int64_t workmem_used; /* Instrumentation.workmemused (bytes) */ + int64_t workmem_wanted; /* Instrumentation.workmemwanted (bytes); >0 == spilled */ + /* + * Derived rate fields, computed in signal_handler from the per-node rolling + * state (previous ntuples and sample time) rather than read from + * Instrumentation. Zero on the node's first sample. + */ + double ntuples_delta; /* tuples produced since the previous sample */ + double tuples_per_sec; /* ntuples_delta divided by the sample interval */ + double time_since_init_sec; /* seconds since the node's first sample */ + bool stalled; /* executing but produced no new tuples and not at eof */ + char relation_name[MAX_RELNAME_LEN]; +} GpscNodeSample; + +#endif /* QS_TYPES_H */ diff --git a/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c b/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c new file mode 100644 index 00000000000..843bbe9a972 --- /dev/null +++ b/gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c @@ -0,0 +1,1000 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * signal_handler.c + * Custom signal handlers and plan-tree walker for pg_query_state. + * + * This module implements the two custom ProcSignal handlers registered by + * pg_qs_init(): + * + * SendQueryState() -- fired when QueryStatePollReason is received. + * Walks the active plan tree, collects per-node stats, + * logs them, then pushes the whole snapshot to the + * yagpcc UDS sink (and, on the coordinator, the + * deparsed plan document). + * SendCdbComponents() -- fired when BackendInfoPollReason is received (QD only). + * Sends the list of active QE (segid, pid) pairs. + * + * Also contains: + * qs_planstate_walker() -- recursive plan-tree traversal helper. + * qs_get_node_stats() -- per-node stat collection callback. + * qs_debug_node_stats() -- LOG-level dump of a collected stat list. + * qs_debug_node_sample() -- LOG-level dump of a single GpscNodeSample. + * send_msg_by_parts() -- chunked shm_mq send helper. + * + * Portions derived from pg_query_state + * (https://github.com/postgrespro/pg_query_state), under the PostgreSQL + * License: + * Portions Copyright (c) 2016-2025, Postgres Professional + * + * IDENTIFICATION + * gpcontrib/gp_stats_collector/src/pg_query_state/signal_handler.c + * + *------------------------------------------------------------------------- + */ + +#include + +#include "pg_query_state.h" +#include "PlanNodeEmitter.h" + +#include "access/xact.h" +#include "cdb/cdbexplain.h" +#include "cdb/cdbutil.h" +#include "cdb/cdbvars.h" +#include "libpq-fe.h" +#include "cdb/cdbconn.h" +#include "commands/explain.h" +#include "executor/executor.h" +#include "miscadmin.h" +#include "nodes/execnodes.h" +#include "nodes/plannodes.h" +#include "pgstat.h" +#include "parser/parsetree.h" +#include "storage/bufmgr.h" +#include "storage/lock.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/resowner.h" +#include "utils/rel.h" +#include "utils/timestamp.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "libpq/pqmq.h" + +/* + * Identity of the most recent coordinator plan-doc push, used to rate-limit + * SetQueryPlanReq: SendQueryState() re-sends the deparsed plan only when the + * query key changes or PLAN_DOC_RESEND_INTERVAL_MS has elapsed. + */ +static struct +{ + int32_t tmid; + int32_t ssid; + int32_t ccnt; + TimestampTz at; +} last_sent_query_key; + +typedef struct NodeRollState +{ + int32_t plan_node_id; + double prev_ntuples_sum; + TimestampTz prev_executed_at; + TimestampTz first_executed_at; + int32_t relation_oid; + char relation_name[MAX_RELNAME_LEN]; +} NodeRollState; + +static HTAB *node_roll_htab = NULL; + +static void ensure_node_roll_htab(void) +{ + HASHCTL ctl; + + if (node_roll_htab) + { + return; + } + + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(int); + ctl.entrysize = sizeof(NodeRollState); + ctl.hcxt = TopMemoryContext; + node_roll_htab = hash_create("gpsc_per_node_roll_state", + 64, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); +} + +void gpsc_reset_node_roll_state(void) +{ + if (node_roll_htab) + { + hash_destroy(node_roll_htab); + node_roll_htab = NULL; + } +} + +/* + * qs_reporting_segid -- segid this backend stamps on the samples it emits. + * + * GpIdentity.segindex is -1 both here and on the QD; an executor role on the + * coordinator host means the entry-db QE, which needs a segid of its own. + */ +static int32 +qs_reporting_segid(void) +{ + if (Gp_role == GP_ROLE_EXECUTE && GpIdentity.segindex < 0) + return GPSC_SEGID_ENTRY_DB; + + return GpIdentity.segindex; +} + +/* + * shm_mq_send_nonblocking -- attempt to send nbytes through mqh up to + * `attempts` times, sleeping WRITING_DELAY µs between retries. + * + * Returns MSG_BY_PARTS_FAILED immediately on SHM_MQ_DETACHED; retries on + * SHM_MQ_WOULD_BLOCK. + */ +static msg_by_parts_result +shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, + const void *data, Size attempts) +{ + int i; + shm_mq_result res; + + for (i = 0; i < (int) attempts; i++) + { +#if PG_VERSION_NUM < 150000 + res = shm_mq_send(mqh, nbytes, data, true); +#else + res = shm_mq_send(mqh, nbytes, data, true, true); +#endif + + if (res == SHM_MQ_SUCCESS) + break; + else if (res == SHM_MQ_DETACHED) + return MSG_BY_PARTS_FAILED; + + /* SHM_MQ_WOULD_BLOCK -- back off briefly and retry. */ + pg_usleep(WRITING_DELAY); + } + + if (i == (int) attempts) + return MSG_BY_PARTS_FAILED; + + return MSG_BY_PARTS_SUCCEEDED; +} + +/* + * send_msg_by_parts -- transmit an arbitrarily large buffer through mqh. + * + * The wire protocol is: first send a Size value announcing the total payload + * length, then send the payload itself in chunks of at most MSG_MAX_SIZE + * bytes. The receiver must use receive_msg_by_parts() (in pg_query_state.c) + * to reassemble the chunks. + * + * Parameters: + * mqh -- attached shm_mq handle (sender side) + * nbytes -- total payload size + * data -- pointer to the payload + * + * Returns MSG_BY_PARTS_SUCCEEDED on success, MSG_BY_PARTS_FAILED otherwise. + */ +static msg_by_parts_result +send_msg_by_parts(shm_mq_handle *mqh, Size nbytes, const void *data) +{ + int offset; + int bytes_left; + int bytes_send; + + /* Announce total length. */ + if (shm_mq_send_nonblocking(mqh, sizeof(Size), &nbytes, + NUM_OF_ATTEMPTS) == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; + + /* Send payload in chunks. */ + for (offset = 0; offset < (int) nbytes; offset += bytes_send) + { + bytes_left = nbytes - offset; + bytes_send = (bytes_left < MSG_MAX_SIZE) ? bytes_left : MSG_MAX_SIZE; + if (shm_mq_send_nonblocking(mqh, bytes_send, + &(((unsigned char *) data)[offset]), + NUM_OF_ATTEMPTS) == MSG_BY_PARTS_FAILED) + return MSG_BY_PARTS_FAILED; + } + + return MSG_BY_PARTS_SUCCEEDED; +} + +/* + * qs_planstate_walker -- depth-first traversal of a PlanState tree. + * + * Visits every node in the tree rooted at `planstate`, calling `executor` + * on each node before recursing. Handles all node types that have child + * plan states (Append, MergeAppend, BitmapAnd/Or, SubqueryScan, CustomScan, + * init-plans, and sub-plans). + * + * Parameters: + * planstate -- root of the subtree to walk (NULL is a no-op) + * executor -- callback invoked for each node + * qs_walker_ctx -- context threaded through all callbacks + * depth -- current recursion depth (for stack-depth checks) + */ +static void +qs_planstate_walker(PlanState *planstate, + qs_planstate_walker_callback executor, + QsWalkerContext *qs_walker_ctx, + int depth) +{ + int32 saved_parent_plan_node_id; + int32 saved_slice_id; + Plan *plan; + ListCell *lc; + + if (planstate == NULL) + return; + + check_stack_depth(); + + plan = planstate->plan; + + /* + * A Motion opens a new slice, and the node itself belongs to the sending + * side -- the same attribution ExplainNode() uses. Switch before sampling + * so the Motion is reported under its own slice, not its parent's. + */ + saved_slice_id = qs_walker_ctx->slice_id; + if (IsA(plan, Motion)) + { + Motion *motion = (Motion *) plan; + SliceTable *sliceTable = planstate->state->es_sliceTable; + + if (sliceTable && motion->motionID >= 0 && + motion->motionID < sliceTable->numSlices) + qs_walker_ctx->slice_id = sliceTable->slices[motion->motionID].sliceIndex; + } + + executor(planstate, qs_walker_ctx); + saved_parent_plan_node_id = qs_walker_ctx->parent_plan_node_id; + qs_walker_ctx->parent_plan_node_id = plan->plan_node_id; + + /* initPlans */ + foreach(lc, planstate->initPlan) + { + SubPlanState *sps = lfirst_node(SubPlanState, lc); + qs_planstate_walker(sps->planstate, executor, qs_walker_ctx, depth + 1); + } + + /* Left and right children. */ + qs_planstate_walker(outerPlanState(planstate), executor, qs_walker_ctx, + depth + 1); + qs_planstate_walker(innerPlanState(planstate), executor, qs_walker_ctx, + depth + 1); + + /* Type-specific child plans. */ + switch (nodeTag(plan)) + { + case T_Append: + { + AppendState *as = (AppendState *) planstate; + for (int i = 0; i < as->as_nplans; i++) + qs_planstate_walker(as->appendplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_MergeAppend: + { + MergeAppendState *ms = (MergeAppendState *) planstate; + for (int i = 0; i < ms->ms_nplans; i++) + qs_planstate_walker(ms->mergeplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_BitmapAnd: + { + BitmapAndState *bas = (BitmapAndState *) planstate; + for (int i = 0; i < bas->nplans; i++) + qs_planstate_walker(bas->bitmapplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_BitmapOr: + { + BitmapOrState *bos = (BitmapOrState *) planstate; + for (int i = 0; i < bos->nplans; i++) + qs_planstate_walker(bos->bitmapplans[i], executor, + qs_walker_ctx, depth + 1); + break; + } + case T_SubqueryScan: + qs_planstate_walker(((SubqueryScanState *) planstate)->subplan, + executor, qs_walker_ctx, depth + 1); + break; + case T_CustomScan: + foreach(lc, ((CustomScanState *) planstate)->custom_ps) + qs_planstate_walker((PlanState *) lfirst(lc), executor, + qs_walker_ctx, depth + 1); + break; + default: + break; + } + + /* subPlans */ + foreach(lc, planstate->subPlan) + { + SubPlanState *sps = lfirst_node(SubPlanState, lc); + qs_planstate_walker(sps->planstate, executor, qs_walker_ctx, depth + 1); + } + + qs_walker_ctx->parent_plan_node_id = saved_parent_plan_node_id; + qs_walker_ctx->slice_id = saved_slice_id; +} + +/* + * qs_get_node_stats -- walker callback that snapshots one plan node. + * + * Allocates a GpscNodeSample in the current memory context, fills it from + * planstate->instrument (if available), and appends it to + * qs_walker_ctx->per_node_stats. + * + * Parameters: + * planstate -- the plan node being sampled + * qs_walker_ctx -- walker context; per_node_stats is extended in-place + */ +static void +qs_get_node_stats(PlanState *planstate, QsWalkerContext *qs_walker_ctx) +{ + GpscNodeSample *nodestat = + (GpscNodeSample *) palloc0(sizeof(GpscNodeSample)); + + /* Identity fields. */ + nodestat->ssid = gp_session_id; + nodestat->tmid = qs_walker_ctx->tmid; + nodestat->ccnt = gp_command_count; + + /* Plan-tree position. */ + nodestat->plan_node_id = planstate->plan->plan_node_id; + nodestat->parent_plan_node_id = qs_walker_ctx->parent_plan_node_id; + nodestat->node_tag = nodeTag(planstate->plan); + nodestat->slice_id = qs_walker_ctx->slice_id; + nodestat->segindex = qs_reporting_segid(); + nodestat->dbid = GpIdentity.dbid; + nodestat->pid = MyProcPid; + + /* Planner estimate. */ + nodestat->plan_rows = planstate->plan->plan_rows; + + /* Runtime instrumentation (may be NULL for non-instrumented nodes). */ + if (planstate->instrument) + { + Instrumentation *instr = planstate->instrument; + double eff_nloops; + + if (qs_walker_ctx->finalize) + { + InstrEndLoop(instr); + } + + eff_nloops = instr->nloops; + if (!qs_walker_ctx->finalize && instr->eof) + eff_nloops += 1; + + nodestat->ntuples = instr->ntuples + instr->tuplecount; /* include in-progress loop */ + nodestat->tuplecount = instr->tuplecount; + nodestat->nloops = eff_nloops; + nodestat->startup = instr->startup; + nodestat->total = instr->total; + nodestat->firsttuple = instr->firsttuple; + + nodestat->shared_blks_hit = instr->bufusage.shared_blks_hit; + nodestat->shared_blks_read = instr->bufusage.shared_blks_read; + + /* + * eof lets a consumer tell a node that has finished producing (running + * but exhausted for this cycle) from one still actively pulling. + */ + nodestat->eof = instr->eof; + + if (instr->running && !instr->eof) + nodestat->node_status = QS_NODE_STATUS_EXECUTING; + else if (eff_nloops > 0) + nodestat->node_status = QS_NODE_STATUS_FINISHED; + else + nodestat->node_status = QS_NODE_STATUS_INITIALIZED; + + nodestat->workfile_created = instr->workfileCreated; + nodestat->workmem_used = (int64_t) instr->workmemused; + nodestat->workmem_wanted = (int64_t) instr->workmemwanted; + } + else + { + nodestat->node_status = QS_NODE_STATUS_INITIALIZED; + } + + qs_walker_ctx->per_node_stats = + lappend(qs_walker_ctx->per_node_stats, nodestat); + + { + TimestampTz ts_now = qs_walker_ctx->ts_now; + double cur_sum = nodestat->ntuples; + bool found; + NodeRollState *rs; + + rs = (NodeRollState *) hash_search(node_roll_htab, + &nodestat->plan_node_id, HASH_ENTER, &found); + + if (found) + { + double dt = (double) (ts_now - rs->prev_executed_at) / USECS_PER_SEC; + nodestat->ntuples_delta = cur_sum - rs->prev_ntuples_sum; + nodestat->tuples_per_sec = (dt > 0) ? nodestat->ntuples_delta / dt : 0; + nodestat->time_since_init_sec = (double) (ts_now - rs->first_executed_at) / USECS_PER_SEC; + + /* Relation identity is invariant per plan node: reuse the cache. */ + nodestat->relation_oid = rs->relation_oid; + strlcpy(nodestat->relation_name, rs->relation_name, MAX_RELNAME_LEN); + } + else + { + Index rti = 0; + + nodestat->ntuples_delta = cur_sum; + nodestat->tuples_per_sec = 0; + nodestat->time_since_init_sec = 0; + rs->first_executed_at = ts_now; + + switch (nodeTag(planstate->plan)) + { + case T_SeqScan: + case T_DynamicSeqScan: + case T_SampleScan: + case T_IndexScan: + case T_DynamicIndexScan: + case T_DynamicIndexOnlyScan: + case T_IndexOnlyScan: + case T_BitmapHeapScan: + case T_DynamicBitmapHeapScan: + case T_TidScan: + case T_TidRangeScan: + case T_ForeignScan: + case T_DynamicForeignScan: + case T_CustomScan: + rti = ((Scan *) planstate->plan)->scanrelid; + break; + case T_ModifyTable: + rti = ((ModifyTable *) planstate->plan)->nominalRelation; + break; + default: + break; + } + + if (rti > 0 && planstate->state) + { + List *rtable = planstate->state->es_range_table; + if (rti <= (Index) list_length(rtable)) + { + RangeTblEntry *rte = rt_fetch(rti, rtable); + if (rte->rtekind == RTE_RELATION) + { + char *relname; + nodestat->relation_oid = (int32_t) rte->relid; + relname = get_rel_name(rte->relid); + if (relname) + { + strlcpy(nodestat->relation_name, relname, MAX_RELNAME_LEN); + pfree(relname); + } + } + } + } + + /* Cache the resolved identity for subsequent polls. */ + rs->relation_oid = nodestat->relation_oid; + strlcpy(rs->relation_name, nodestat->relation_name, MAX_RELNAME_LEN); + } + + nodestat->stalled = (nodestat->ntuples_delta == 0 + && nodestat->node_status == QS_NODE_STATUS_EXECUTING + && !nodestat->eof); + rs->prev_ntuples_sum = cur_sum; + rs->prev_executed_at = ts_now; + } +} + +/* + * qs_debug_node_sample -- emit a single GpscNodeSample to the PostgreSQL LOG. + * + * Intended for development and integration testing. In production deployments + * this will produce a large number of log lines; suppress with log_min_messages. + */ +static void +qs_debug_node_sample(GpscNodeSample *s) +{ + elog(DEBUG1, + "GpscNodeSample: " + "plan_node_id=%d parent=%d node_tag=%d " + "slice_id=%d segindex=%d " + "tmid=%d ssid=%d ccnt=%d " + "plan_rows=%.0f " + "ntuples=%.0f tuplecount=%.0f nloops=%.0f " + "startup=%f total=%f firsttuple=%f " + "shared_blks_hit=%lu shared_blks_read=%lu " + "workfile_created=%d workmem_used=%ld workmem_wanted=%ld " + "node_status=%d", + s->plan_node_id, s->parent_plan_node_id, s->node_tag, + s->slice_id, s->segindex, + s->tmid, s->ssid, s->ccnt, + s->plan_rows, + s->ntuples, s->tuplecount, s->nloops, + s->startup, s->total, s->firsttuple, + s->shared_blks_hit, s->shared_blks_read, + (int) s->workfile_created, (long) s->workmem_used, (long) s->workmem_wanted, + (int) s->node_status); +} + +/* + * qs_debug_node_stats -- emit all nodes in per_node_stats to the PostgreSQL LOG. + * + * Logs a summary line followed by one line per node via qs_debug_node_sample(). + */ +static void +qs_debug_node_stats(List *per_node_stats) +{ + ListCell *lc; + int i = 0; + + if (!message_level_is_interesting(DEBUG1)) + return; + + elog(DEBUG1, "GpscNodeSample list: %d nodes", list_length(per_node_stats)); + foreach(lc, per_node_stats) + { + GpscNodeSample *s = (GpscNodeSample *) lfirst(lc); + elog(DEBUG1, "--- node[%d] ---", i++); + qs_debug_node_sample(s); + } +} + +/* + * runtime_explain -- snapshot the active query's plan tree. + * + * Retrieves the top-most QueryDesc from QueryDescStack, walks its planstate + * tree with qs_get_node_stats(), and returns the resulting List of + * GpscNodeSample pointers. + * + * Callers must ensure QueryDescStack is non-empty before calling this. + */ +static List * +runtime_explain(TimestampTz ts_now) +{ + QsWalkerContext *qs_walker_ctx = + (QsWalkerContext *) palloc0(sizeof(QsWalkerContext)); + QueryDesc *queryDesc; + + Assert(list_length(QueryDescStack) > 0); + queryDesc = get_toppest_query(); + qs_walker_ctx->ts_now = ts_now; + qs_walker_ctx->parent_plan_node_id = GPSC_NO_PARENT_PLAN_NODE_ID; + qs_walker_ctx->slice_id = queryDesc->estate + ? LocallyExecutingSliceIndex(queryDesc->estate) + : currentSliceId; + gp_gettmid(&qs_walker_ctx->tmid); + ensure_node_roll_htab(); + qs_planstate_walker(queryDesc->planstate, qs_get_node_stats, + qs_walker_ctx, 0); + return qs_walker_ctx->per_node_stats; +} + +/* + * emit_node_batch -- push a whole plan-tree snapshot as one SetPerNodeBatchReq. + * + * Flattens the List into a contiguous array and hands it to + * the C++ emitter, which opens a single UDS connection for the whole backend + * instead of one connection per node. A NULL or empty list is a no-op. + * + * The caller is responsible for calling gpsc_qs_sync_config() beforehand. + */ +static void +emit_node_batch(List *per_node_stats, const char *trace_id) +{ + GpscNodeSample **arr; + ListCell *lc; + int n = list_length(per_node_stats); + int i = 0; + + if (n == 0) + return; + + arr = (GpscNodeSample **) palloc(n * sizeof(GpscNodeSample *)); + foreach(lc, per_node_stats) + arr[i++] = (GpscNodeSample *) lfirst(lc); + + gpsc_emit_node_batch(arr, n, trace_id); +} + +/* + * build_plan_doc -- render the active query's plan via ExplainPrintPlan. + * + * Produces the full deparsed plan document (expressions, costs, Settings) in + * the requested ExplainFormat. ExplainBeginOutput/ExplainEndOutput and the + * enclosing "Query" group frame the output so JSON/XML/YAML come out + * well-formed: ExplainPrintPlan on its own renders only the inner "Plan" + * property, so without the group the non-text formats are an unwrapped + * fragment no parser accepts. The framing lives here, outside + * ExplainPrintPlan, so that function is left untouched. + * + * Returns a palloc'd string in the current context, or NULL when queryDesc is + * NULL. Intended for the coordinator (QD) only: on a QE the plan subtree can + * reach child PlanStates from other slices that are not instantiated here. + */ +static char * +build_plan_doc(QueryDesc *queryDesc, ExplainFormat format) +{ + ExplainState *es; + + if (queryDesc == NULL) + return NULL; + + HOLD_INTERRUPTS(); + { + es = NewExplainState(); + es->format = format; + es->verbose = true; + es->costs = true; + es->runtime = true; + ExplainBeginOutput(es); + ExplainOpenGroup("Query", NULL, true, es); + ExplainPrintPlan(es, queryDesc); + ExplainCloseGroup("Query", NULL, true, es); + ExplainEndOutput(es); + } + RESUME_INTERRUPTS(); + + return es->str->data; +} + +/* + * SendQueryState -- handler for QueryStatePollReason. + * + * Fired asynchronously when another backend (or the monitoring function) + * sends QueryStatePollReason to this process. + * + * Collects a plan-tree snapshot via runtime_explain(), logs it via + * qs_debug_node_stats(), then syncs the emitter config and pushes the whole + * snapshot to the yagpcc UDS sink via emit_node_batch(). On the coordinator + * it additionally pushes the deparsed plan document (SetQueryPlanReq), which + * the compact per-node stats cannot reconstruct; that push is rate-limited to + * once per PLAN_DOC_RESEND_INTERVAL_MS per query. + * + * The entire body runs inside a dedicated MemoryContext that is deleted on + * exit, preventing any leaks into the backend's long-lived contexts. Any + * errors are swallowed with FlushErrorState() to avoid crashing the backend. + */ +void +SendQueryState(void) +{ + int saved_errno = errno; + MemoryContext volatile oldcontext = CurrentMemoryContext; + MemoryContext volatile qs_context = NULL; + QueryDesc *qd; + + if (!pg_qs_enable) + { + errno = saved_errno; + return; + } + + if (!list_length(QueryDescStack)) + { + errno = saved_errno; + return; + } + + if (MyBackendId < 1 || MyBackendId > MaxBackends) + { + errno = saved_errno; + return; + } + + if (stack_is_too_deep()) + { + elog(DEBUG1, "pg_query_state: skipping poll, call stack too deep"); + errno = saved_errno; + return; + } + + qd = get_toppest_query(); + if (qd == NULL || qd->planstate == NULL || qd->estate == NULL) + { + errno = saved_errno; + return; + } + + HOLD_INTERRUPTS(); + PG_TRY(); + { + List *qs_result; + TimestampTz now = GetCurrentTimestamp(); + + qs_context = AllocSetContextCreate(TopMemoryContext, + "pg_query_state signal context", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(qs_context); + + qs_result = runtime_explain(now); + qs_debug_node_stats(qs_result); + gpsc_qs_sync_config(); + emit_node_batch(qs_result, qs_trace_slots[MyBackendId]); + + if (Gp_role == GP_ROLE_DISPATCH && + IsTransactionState() && CurrentResourceOwner != NULL) + { + bool is_same_query; + bool is_stale; + int32_t tmid; + + gp_gettmid(&tmid); + is_same_query = (tmid == last_sent_query_key.tmid && + gp_session_id == last_sent_query_key.ssid && + gp_command_count == last_sent_query_key.ccnt); + + is_stale = !is_same_query || + TimestampDifferenceExceeds(last_sent_query_key.at, + now, + PLAN_DOC_RESEND_INTERVAL_MS); + + if (is_stale) + { + char *plan_doc = build_plan_doc(qd, EXPLAIN_FORMAT_JSON); + + gpsc_emit_query_plan(tmid, gp_session_id, gp_command_count, + plan_doc, EXPLAIN_FORMAT_JSON); + + last_sent_query_key.tmid = tmid; + last_sent_query_key.ssid = gp_session_id; + last_sent_query_key.ccnt = gp_command_count; + last_sent_query_key.at = now; + } + } + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcontext); + + if (!elog_dismiss(WARNING)) + { + if (qs_context) + MemoryContextDelete(qs_context); + RESUME_INTERRUPTS(); + errno = saved_errno; + PG_RE_THROW(); + } + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldcontext); + if (qs_context) + MemoryContextDelete(qs_context); + RESUME_INTERRUPTS(); + errno = saved_errno; +} + +/* + * fill_segpid -- append (segid, pid) pairs from one CDB segment's activelist. + * + * msg->pids[] has room for exactly 'cap' entries in total (not 'cap' more). + * *index is the running write position, shared across all calls for one + * message; it is advanced past every entry actually written. + * + * Entries are skipped when the descriptor has no live backend pid yet, so the + * final *index may be LESS than the capacity estimated by the caller. The + * caller must derive both msg->number and msg->length from the final *index, + * never from the estimate. + * + * `is_entry_db` selects which of cdbs->{segment_db_info,entry_db_info} the + * caller is walking. An entry-db descriptor carries segindex -1, the same + * value the QD itself reports, so its entries go out as GPSC_SEGID_ENTRY_DB. + * + * Returns true if the capacity was hit and one or more writable entries were + * dropped. + */ +static bool +fill_segpid(CdbComponentDatabaseInfo *segInfo, backend_info *msg, Size cap, + Size *index, bool is_entry_db) +{ + ListCell *lc; + gp_segment_pid *segpid; + SegmentDatabaseDescriptor *dbdesc; + + foreach(lc, segInfo->activelist) + { + dbdesc = (SegmentDatabaseDescriptor *) lfirst(lc); + if (!dbdesc || dbdesc->backendPid <= 0) + continue; + + if (!is_entry_db && dbdesc->segindex < 0) + continue; + + if (*index >= cap) + return true; + + segpid = &msg->pids[(*index)++]; + segpid->pid = dbdesc->backendPid; + segpid->segid = is_entry_db ? GPSC_SEGID_ENTRY_DB : dbdesc->segindex; + } + + return false; +} + +static int +count_active(CdbComponentDatabaseInfo *dbs, Size n) +{ + int cnt = 0; + for (Size i = 0; i < n; ++i) + { + cnt += list_length(dbs[i].activelist); + } + + return cnt; +} + +/* + * SendCdbComponents -- handler for BackendInfoPollReason (QD only). + * + * Collects the list of active QE (segid, pid) pairs from the CDB component + * database and sends them back to the requestor through shm_mq as a + * backend_info message. + * + * Side effects: + * - Calls cdbcomponent_getCdbComponents(); the returned structure is owned + * and cached by the CDB component cache (CdbComponentsContext), NOT by + * the local context below, and must not be freed here. + * - Only the locally built backend_info message is allocated in the + * short-lived context, which is deleted on every exit path. + */ +void +SendCdbComponents(void) +{ + int saved_errno = errno; + shm_mq_handle *volatile mqh = NULL; + CdbComponentDatabases *cdbs; + MemoryContext volatile oldctx = CurrentMemoryContext; + MemoryContext volatile ctx = NULL; + Size index = 0; + msg_by_parts_result send_result = MSG_BY_PARTS_SUCCEEDED; + + if (!mq || shm_mq_get_sender(mq) != MyProc || !mq_req_id) + { + errno = saved_errno; + return; + } + + if (!params || params->reason != BackendInfoPollReason) + { + errno = saved_errno; + return; + } + + HOLD_INTERRUPTS(); + PG_TRY(); + { + ctx = AllocSetContextCreate(TopMemoryContext, + "pg_query_state SendCdbComponents", ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(ctx); + + mqh = shm_mq_attach(mq, NULL, NULL); + + if (Gp_role != GP_ROLE_DISPATCH) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: running not on QD"); + shm_mq_msg error_msg = {*mq_req_id, BASE_SIZEOF_SHM_MQ_MSG, + MyProc, WRONG_ROLE}; + send_result = send_msg_by_parts(mqh, error_msg.length, &error_msg); + } + else if (!pg_qs_enable) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: module disabled"); + shm_mq_msg disabled_msg = {*mq_req_id, BASE_SIZEOF_SHM_MQ_MSG, + MyProc, STAT_DISABLED}; + send_result = send_msg_by_parts(mqh, disabled_msg.length, &disabled_msg); + } + else if (list_length(QueryDescStack) == 0) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: no active query"); + shm_mq_msg not_running_msg = {*mq_req_id, BASE_SIZEOF_SHM_MQ_MSG, + MyProc, QUERY_NOT_RUNNING}; + send_result = send_msg_by_parts(mqh, not_running_msg.length, ¬_running_msg); + } + else + { + MemoryContextSwitchTo(oldctx); + cdbs = cdbcomponent_getCdbComponents(); + MemoryContextSwitchTo(ctx); + + int qecount = count_active(cdbs->entry_db_info, cdbs->total_entry_dbs) + + count_active(cdbs->segment_db_info, cdbs->total_segment_dbs); + + size_t bufsz = BASE_SIZEOF_GP_BACKEND_INFO + sizeof(gp_segment_pid) * qecount; + backend_info *msg = (backend_info *) palloc0(bufsz); + + bool truncated = false; + + for (int i = 0; i < cdbs->total_segment_dbs; ++i) + { + CdbComponentDatabaseInfo *segInfo = &cdbs->segment_db_info[i]; + truncated |= fill_segpid(segInfo, msg, qecount, &index, false); + } + + for (int i = 0; i < cdbs->total_entry_dbs; ++i) + { + CdbComponentDatabaseInfo *segInfo = &cdbs->entry_db_info[i]; + truncated |= fill_segpid(segInfo, msg, qecount, &index, true); + } + + if (truncated) + { + elog(WARNING, "pg_query_state: SendCdbComponents: backend list truncated at %d of %d entries", + (int) index, qecount); + } + + msg->reqid = *mq_req_id; + msg->length = BASE_SIZEOF_GP_BACKEND_INFO + sizeof(gp_segment_pid) * index; + msg->result_code = QS_RETURNED; + Assert(index <= qecount); + msg->number = index; + send_result = send_msg_by_parts(mqh, msg->length, msg); + } + + if (send_result != MSG_BY_PARTS_SUCCEEDED) + { + elog(DEBUG1, "pg_query_state: SendCdbComponents: send failed (%d)", + (int) send_result); + } + + shm_mq_detach(mqh); + mqh = NULL; + } + PG_CATCH(); + { + if (mqh) + { + shm_mq_detach(mqh); + mqh = NULL; + } + MemoryContextSwitchTo(oldctx); + + if (!elog_dismiss(WARNING)) + { + if (ctx) + MemoryContextDelete(ctx); + + RESUME_INTERRUPTS(); + errno = saved_errno; + PG_RE_THROW(); + } + } + PG_END_TRY(); + + MemoryContextSwitchTo(oldctx); + if (ctx) + MemoryContextDelete(ctx); + + RESUME_INTERRUPTS(); + errno = saved_errno; +} diff --git a/gpcontrib/gp_stats_collector/test/Makefile b/gpcontrib/gp_stats_collector/test/Makefile new file mode 100644 index 00000000000..3931f4d9592 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/Makefile @@ -0,0 +1,22 @@ +# Regression tests for the gp_stats_collector pg_query_state signal API. +# +# Self-contained installcheck suite: the extension must already be built and +# installed (make -C .. install) and loaded via shared_preload_libraries in the +# target cluster. Run with: +# +# make -C gpcontrib/gp_stats_collector/test installcheck +# +# pg_regress defaults to ./sql/.sql and ./expected/.out. + +REGRESS = gpsc_pg_query_state + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = gpcontrib/gp_stats_collector/test +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/gpcontrib/gp_stats_collector/test/crash/README.md b/gpcontrib/gp_stats_collector/test/crash/README.md new file mode 100644 index 00000000000..1ecdb0a376f --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/crash/README.md @@ -0,0 +1,38 @@ +# gp_stats_collector crash test + +Liveness test: with the runtime query-state feature fully enabled and a poller +tracing every running query, Cloudberry must not crash and queries must still +finish with the same results as without the feature. + +Driven by `.github/workflows/gpsc-crash-test.yaml` (manual / nightly). One build, +one demo cluster, two `installcheck-parallel` passes on it: + +1. **baseline** — feature OFF (stock Cloudberry, module not preloaded) → record + failed tests. +2. **traced** — feature ON + poller running → record failed tests, then the + crash gate. + +**Hard verdict: the crash gate** (no PANIC / signal / segment down / dead +coordinator). The failed-test delta `traced \ baseline` is reported for +information only and does **not** fail the job: `installcheck-parallel` is not +diff-deterministic, so a tracing-only failure is not, by itself, a regression — +inspect the uploaded `run2-traced.diffs` by hand. + +Workload is `installcheck-parallel` (upstream `parallel_schedule`): fast and +fault-free. Because it injects no faults, any PANIC in the logs is a genuine +crash, which keeps the crash gate simple and honest. + +## Files + +- `poller.py` — single-process tracer: loops over active client backends in + `pg_stat_activity` and calls `gpsc.pg_query_state(pid, trace_id)` on each, with + a per-pid cooldown so no pid is polled while a prior poll is in flight (the + extension does not support overlapping polls of one pid). Uses `psql`, no + Python DB driver. +- `uds_drain.py` — minimal `AF_UNIX` sink for `gpsc.uds_path`; reads and discards + so the serialize+send path runs without the real yagpcc. +- `extract_failures.sh` — pulls the set of `... FAILED` test names from a + `make installcheck-parallel` log. +- `crash_scan.sh` — the crash gate: log crash markers, `gpstate -e`, `SELECT 1`. + +Design notes: `../../docs/gpsc-crash-test-design.md` (local, not committed). diff --git a/gpcontrib/gp_stats_collector/test/crash/crash_scan.sh b/gpcontrib/gp_stats_collector/test/crash/crash_scan.sh new file mode 100755 index 00000000000..9b89f904e82 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/crash/crash_scan.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# crash_scan.sh +# +# The crash gate for the tracer run. Independent of test diffs: it decides +# whether Cloudberry survived the tracing. Exits non-zero (and prints why) on +# any of: +# - a crash marker in a coordinator/segment log under ; +# - a segment reported down / resyncing by gpstate -e; +# - the coordinator failing to answer a trivial query. +# +# The demo cluster env (gpdemo-env.sh) must be sourced before calling. +# -------------------------------------------------------------------- +set -uo pipefail + +log_root="${1:?usage: crash_scan.sh }" +status=0 + +# Unambiguous crash markers only. The workload (installcheck-parallel) injects +# no faults, so a PANIC / signal here is a genuine crash, not a fault-injection +# recovery test. Excluded on purpose: +# - plain FATAL: routine during regression (missing role, duplicate object). +# - "server closed the connection unexpectedly" / "the database system is in +# recovery mode": routine mirror/walreceiver churn on every restart +# (gpstop -ar), not a crash. +# Real crashes are caught here (PANIC, postmaster-wide crash restart, a process +# killed by a signal) and corroborated by gpstate -e + SELECT 1 below. +patterns='PANIC|terminating connection because of crash of another server process|was terminated by signal [0-9]' + +echo "== crash_scan: log markers under ${log_root} ==" +if hits=$(grep -rERn "${patterns}" "${log_root}" 2>/dev/null); then + if [ -n "${hits}" ]; then + echo "CRASH: crash markers found:" + echo "${hits}" | head -50 + status=1 + fi +fi +[ "${status}" -eq 0 ] && echo " none" + +echo "== crash_scan: segment health (gpstate -e) ==" +if command -v gpstate >/dev/null 2>&1; then + gpstate_out=$(gpstate -e 2>&1 || true) + echo "${gpstate_out}" | tail -30 + if echo "${gpstate_out}" | grep -Eiq 'down|resynchroniz|not synchronized|Unsynchronized'; then + echo "CRASH: gpstate reports segments down / resyncing" + status=1 + fi +else + echo " gpstate not on PATH (env not sourced?)" + status=1 +fi + +echo "== crash_scan: coordinator responsive? ==" +if echo 'SELECT 1;' | psql -X -q -A -t -d postgres >/dev/null 2>&1; then + echo " SELECT 1 ok" +else + echo "CRASH: coordinator did not answer SELECT 1" + status=1 +fi + +if [ "${status}" -eq 0 ]; then + echo "== crash_scan: PASS (cluster healthy) ==" +else + echo "== crash_scan: FAIL (see markers above) ==" +fi +exit "${status}" diff --git a/gpcontrib/gp_stats_collector/test/crash/extract_failures.sh b/gpcontrib/gp_stats_collector/test/crash/extract_failures.sh new file mode 100755 index 00000000000..2a22fe21363 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/crash/extract_failures.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# extract_failures.sh +# +# Prints the sorted, unique set of test names that pg_regress / isolation2 +# reported as FAILED in an installcheck-world make log, one per line. Lines +# look like "test foo ... FAILED" or " foo ... FAILED"; the test name is +# the token immediately before the "..." separator. +# -------------------------------------------------------------------- +set -euo pipefail + +log="${1:?usage: extract_failures.sh }" + +awk ' + /\.\.\.[[:space:]]*FAILED/ { + for (i = 1; i <= NF; i++) + if ($i == "...") { print $(i - 1); break } + } +' "${log}" | sort -u diff --git a/gpcontrib/gp_stats_collector/test/crash/poller.py b/gpcontrib/gp_stats_collector/test/crash/poller.py new file mode 100755 index 00000000000..f21b234d103 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/crash/poller.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# poller.py -- crash-test tracer for gp_stats_collector. +# +# A single process that, in a loop, finds every active client backend in +# pg_stat_activity and calls gpsc.pg_query_state(pid, trace_id) against it -- +# "trace everything that moves" -- while a heavy test suite runs concurrently. +# +# Deliberately single-process: the extension does not support overlapping polls +# of the same pid, so no pid is ever polled from two sessions at once. A +# per-pid cooldown keeps a gap larger than a collection's latency, so even this +# one session never re-fires a pid whose previous poll may still be in flight. +# +# Errors from a poll (backend gone, permission gate, races) are swallowed on +# purpose: this test cares about cluster liveness, not trace correctness. +# +# Stops when the --stop-file appears. Talks to the server through psql, so it +# needs no Python database driver; the gpdemo environment must be sourced first +# (PGPORT etc.). + +import argparse +import os +import secrets +import subprocess +import sys +import time + +APP_NAME = "gpsc_crash_poller" + +# Backends carrying this application_name are our own psql calls; never trace +# them, or the poller would chase its own tail. +ACTIVE_PIDS_SQL = ( + "SELECT pid FROM pg_stat_activity " + "WHERE state = 'active' " + "AND backend_type = 'client backend' " + "AND coalesce(application_name, '') <> '{app}' " + "AND pid <> pg_backend_pid();" +).format(app=APP_NAME) + + +def psql(dbname, sql, timeout): + """Run one SQL statement through psql; return (rc, stdout). Never raises.""" + env = dict(os.environ, PGAPPNAME=APP_NAME) + try: + proc = subprocess.run( + ["psql", "-X", "-q", "-A", "-t", "-d", dbname, "-c", sql], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + text=True, + ) + return proc.returncode, proc.stdout + except subprocess.TimeoutExpired: + return -1, "psql timeout" + except Exception as exc: # noqa: BLE001 -- liveness test, swallow everything + return -1, str(exc) + + +def main(): + ap = argparse.ArgumentParser(description="gp_stats_collector crash-test poller") + ap.add_argument("--dbname", default="postgres", + help="stable DB to read pg_stat_activity from (default: postgres)") + ap.add_argument("--stop-file", required=True, + help="poller exits once this path exists") + ap.add_argument("--cooldown", type=float, default=1.0, + help="min seconds between two polls of the same pid") + ap.add_argument("--round-sleep", type=float, default=0.05, + help="seconds to sleep between scan rounds") + ap.add_argument("--call-timeout", type=float, default=10.0, + help="per-psql-call timeout in seconds") + ap.add_argument("--log-every", type=int, default=100, + help="print a heartbeat every N rounds") + args = ap.parse_args() + + last_polled = {} + rounds = 0 + polls = 0 + errors = 0 + logged_error = False # print the first poll error body once, for diagnosis + started = time.monotonic() + + print("poller: start (app_name={}, cooldown={}s)".format(APP_NAME, args.cooldown), + flush=True) + + while not os.path.exists(args.stop_file): + rounds += 1 + rc, out = psql(args.dbname, ACTIVE_PIDS_SQL, args.call_timeout) + if rc != 0: + # The coordinator may be momentarily busy/restarting a session; the + # crash gate, not the poller, decides whether that is fatal. + errors += 1 + time.sleep(args.round_sleep) + continue + + now = time.monotonic() + for line in out.splitlines(): + pid = line.strip() + if not pid: + continue + if now - last_polled.get(pid, 0.0) < args.cooldown: + continue + trace_hex = secrets.token_hex(16) # exactly 16 bytes -> bytea + sql = ("SELECT gpsc.pg_query_state({pid}, '\\x{tid}'::bytea);" + .format(pid=pid, tid=trace_hex)) + prc, pout = psql(args.dbname, sql, args.call_timeout) + last_polled[pid] = now + polls += 1 + if prc != 0: + errors += 1 # gate/race/backend-gone: expected, not fatal here + if not logged_error: + # A wall of errors usually means a setup problem (e.g. the + # function is missing); surface the first one so the log is + # not opaque. + print("poller: first poll error: {}".format(pout.strip()), + flush=True) + logged_error = True + + if rounds % args.log_every == 0: + print("poller: rounds={} polls={} errors={} tracked_pids={} elapsed={:.0f}s" + .format(rounds, polls, errors, len(last_polled), + time.monotonic() - started), + flush=True) + + time.sleep(args.round_sleep) + + print("poller: stop (rounds={} polls={} errors={} elapsed={:.0f}s)" + .format(rounds, polls, errors, time.monotonic() - started), + flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gpcontrib/gp_stats_collector/test/crash/uds_drain.py b/gpcontrib/gp_stats_collector/test/crash/uds_drain.py new file mode 100755 index 00000000000..39312ebea92 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/crash/uds_drain.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# uds_drain.py -- minimal Unix-domain-socket sink for gp_stats_collector. +# +# gpsc.logging_mode = UDS makes every backend push per-node batches and plan +# docs to gpsc.uds_path. In the crash test we do not run the real yagpcc; this +# drain accepts every connection and reads/discards whatever arrives, so the +# full C++ serialize+send path runs without backpressure drops -- but nothing +# downstream is exercised or asserted. +# +# One drain covers a single-host demo cluster (all segments share the socket +# path). Runs until killed. + +import argparse +import os +import socket +import sys +import threading + + +def drain_conn(conn): + with conn: + while True: + try: + if not conn.recv(65536): + return + except OSError: + return + + +def main(): + ap = argparse.ArgumentParser(description="gp_stats_collector UDS drain") + ap.add_argument("--path", required=True, help="unix socket path to listen on") + ap.add_argument("--backlog", type=int, default=128) + args = ap.parse_args() + + if os.path.exists(args.path): + os.unlink(args.path) + + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + srv.bind(args.path) + srv.listen(args.backlog) + os.chmod(args.path, 0o777) # any backend user must be able to connect + print("uds_drain: listening on {}".format(args.path), flush=True) + + try: + while True: + conn, _ = srv.accept() + threading.Thread(target=drain_conn, args=(conn,), daemon=True).start() + except KeyboardInterrupt: + pass + finally: + srv.close() + if os.path.exists(args.path): + os.unlink(args.path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out b/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out new file mode 100644 index 00000000000..c3eb3e535d9 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/expected/gpsc_pg_query_state.out @@ -0,0 +1,57 @@ +-- pg_query_state signal API (extension v1.2): catalog contract + negative paths. +-- +-- Deterministic coverage only: SQL-visible function/type registration and the +-- input-validation error branches. The asynchronous happy path (poll a live +-- query and observe per-node stats) is exercised separately under isolation2, +-- since it depends on a second running backend and timing. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore +-- +-- Catalog contract: the three SQL-visible functions are registered in the gpsc +-- schema with the expected return type and dispatch (exec) location. +-- proexeclocation: c = coordinator, a = any (QE-local), s = all segments. +-- +SELECT proname, + pronargs, + prorettype::regtype AS returns, + proexeclocation +FROM pg_proc +WHERE pronamespace = 'gpsc'::regnamespace + AND proname IN ('pg_query_state', 'pg_query_state_backends', 'cbdb_mpp_query_state') +ORDER BY proname; + proname | pronargs | returns | proexeclocation +-------------------------+----------+---------+----------------- + cbdb_mpp_query_state | 2 | void | a + pg_query_state | 2 | void | c + pg_query_state_backends | 1 | record | c +(3 rows) + +-- Composite identifier type used by the signal layer is present. +SELECT typname +FROM pg_type +WHERE typnamespace = 'gpsc'::regnamespace + AND typname = 'gp_segment_pid'; + typname +---------------- + gp_segment_pid +(1 row) + +-- +-- Negative: a backend cannot poll its own state. +-- +SELECT gpsc.pg_query_state(pg_backend_pid(), '\x00112233445566778899aabbccddeeff'::bytea); +ERROR: cannot extract state of current process +SELECT * FROM gpsc.pg_query_state_backends(pg_backend_pid()); +ERROR: cannot extract state of current process +-- +-- Negative: a pid that maps to no live backend is rejected. +-- +SELECT gpsc.pg_query_state(-1, '\x00112233445566778899aabbccddeeff'::bytea); +ERROR: backend with pid=-1 not found +SELECT * FROM gpsc.pg_query_state_backends(-1); +ERROR: backend with pid=-1 not found +-- Cleanup +-- start_ignore +DROP EXTENSION gp_stats_collector; +-- end_ignore diff --git a/gpcontrib/gp_stats_collector/test/isolation2/.gitignore b/gpcontrib/gp_stats_collector/test/isolation2/.gitignore new file mode 100644 index 00000000000..0d2848e26fb --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/.gitignore @@ -0,0 +1,4 @@ +/sql_isolation_testcase.py +/results/ +/regression.diffs +/regression.out diff --git a/gpcontrib/gp_stats_collector/test/isolation2/Makefile b/gpcontrib/gp_stats_collector/test/isolation2/Makefile new file mode 100644 index 00000000000..2a835922c65 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/Makefile @@ -0,0 +1,33 @@ +# isolation2 suite for the gp_stats_collector pg_query_state signal API. +# +# Multi-session tests that a plain pg_regress run cannot express (one backend +# polling another). Reuses the core pg_isolation2_regress harness rather than +# rebuilding it. +# +# Prerequisites (handled by the CI isolation2 job): +# - gp_inject_fault available (--enable-faultinjector, on by default) for the +# happy-path spec. +# - Extension installed and gp_stats_collector in shared_preload_libraries. +# - Harness built: make -C $(top_builddir)/src/test/isolation2 install +# +# Run: +# make -C gpcontrib/gp_stats_collector/test/isolation2 installcheck + +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +ISO2 = $(top_builddir)/src/test/isolation2 + +# isolation2_main.c hardcodes "python3 ./sql_isolation_testcase.py", resolved +# from the current directory, so symlink the core driver here before running. +installcheck: + @ln -sf $(ISO2)/sql_isolation_testcase.py ./sql_isolation_testcase.py + $(ISO2)/pg_isolation2_regress \ + --init-file=$(top_builddir)/src/test/regress/init_file \ + --init-file=$(ISO2)/init_file_isolation2 \ + --inputdir=. --outputdir=. \ + --bindir='$(bindir)' \ + --schedule=./isolation2_schedule + +clean: + rm -rf results/ regression.diffs regression.out sql_isolation_testcase.py diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out new file mode 100644 index 00000000000..a19477a411b --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_backends.out @@ -0,0 +1,27 @@ +-- pg_query_state_backends against an *idle* backend returns an empty set. +-- +-- Deterministic multi-session check with no async race: session 1 tags itself +-- and sits idle; session 2 looks up its pid and polls it. An idle backend is +-- "not running a query", so GetRemoteBackendInfo returns QUERY_NOT_RUNNING and +-- the function yields an empty set (not an error). +-- +-- Extensions are created by setup.sql. + +-- Session 1: tag connection so session 2 can find its pid, then go idle. +1: SET application_name TO 'qs_idle_target'; +SET +1: SELECT 1; + ?column? +---------- + 1 +(1 row) + +-- Session 2: idle target -> zero participating backends. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_idle_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + n_backends +------------ + 0 +(1 row) + +1q: ... +2q: ... diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out new file mode 100644 index 00000000000..792610bb011 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_disabled.out @@ -0,0 +1,61 @@ +-- STAT_DISABLED: when the target backend has pg_query_state.enable = off, its +-- SendCdbComponents reply is STAT_DISABLED, so polling reports an empty backend +-- list even though a query is actively running on the segments. +-- +-- Distinguishes "disabled" from "idle": here the query really is executing +-- (suspended on a fault), yet the disabled module yields nothing. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_disabled_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_disabled_t SELECT generate_series(1, 100); +INSERT 100 + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- Target disables the module for its own session, then runs a query that hangs. +1: SET application_name TO 'qs_disabled_target'; +SET +1: SET pg_query_state.enable TO off; +SET +1&: SELECT count(*) FROM qs_disabled_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Running, but module disabled on the target -> empty backend list. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_disabled_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + n_backends +------------ + 0 +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_disabled_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out new file mode 100644 index 00000000000..8e8f24e6a41 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_perms.out @@ -0,0 +1,92 @@ +-- Permission gate: a non-superuser that does not own the target query is +-- denied; a superuser is allowed. +-- +-- The gate is superuser() || GetUserId() == proc->roleId. isolation2 runs all +-- sessions under the same session role (the one that launched the harness), so +-- the "non-super owner is allowed" branch cannot be expressed here and is not +-- covered; the deny and superuser-allow branches are. +-- +-- A non-superuser cannot see another backend's application_name in +-- pg_stat_activity, so the target pid is captured (as superuser) into a table +-- before SET ROLE. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_perm_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_perm_t SELECT generate_series(1, 100); +INSERT 100 +CREATE TABLE qs_perm_pid (pid int); +CREATE +CREATE ROLE qs_unpriv; +CREATE +GRANT SELECT ON qs_perm_pid TO qs_unpriv; +GRANT +-- No gpsc grants here on purpose: the extension grants USAGE/EXECUTE to PUBLIC +-- in its migration, so an ordinary role reaches the roleId gate exactly as it +-- would in production. This test verifies the gate, not the schema grants. + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +1: SET application_name TO 'qs_perm_target'; +SET +1&: SELECT count(*) FROM qs_perm_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Capture the target pid as superuser (sees application_name). +2: INSERT INTO qs_perm_pid SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_perm_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1; +INSERT 1 + +-- Non-superuser, non-owner: both entry points are denied. +2: SET ROLE qs_unpriv; +SET +2: SELECT gpsc.pg_query_state((SELECT pid FROM qs_perm_pid), '\x00112233445566778899aabbccddeeff'::bytea); +ERROR: permission denied +2: SELECT * FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); +ERROR: permission denied +2: RESET ROLE; +RESET + +-- Superuser: allowed (non-empty backend list). +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); + has_backends +-------------- + t +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP OWNED BY qs_unpriv; +DROP +DROP ROLE qs_unpriv; +DROP +DROP TABLE qs_perm_pid; +DROP +DROP TABLE qs_perm_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out new file mode 100644 index 00000000000..cbced86b14d --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_running.out @@ -0,0 +1,69 @@ +-- Happy path: a query suspended mid-execution on the QEs is observed live. +-- +-- Session 1 launches a query that hits an 'executor_pre_tuple_processed' +-- suspend fault on every primary segment, so its QE backends sit inside the +-- executor with a live plan tree. Session 2 then: +-- * pg_query_state_backends(pid) -> at least one participating backend, +-- * pg_query_state(pid) -> succeeds (fire-and-forget, returns void). +-- The fault is reset and the suspended query is reaped. +-- +-- Extensions (gp_stats_collector, gp_inject_fault) come from setup.sql. + +CREATE TABLE qs_running_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_running_t SELECT generate_series(1, 100); +INSERT 100 + +-- Suspend execution on all primary segments. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- Session 1: tag the connection, then launch a query that hangs on the QEs. +1: SET application_name TO 'qs_running_target'; +SET +1&: SELECT count(*) FROM qs_running_t; + +-- Wait until the fault has been hit on the segments. +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- Session 2: the running query has live QE backends, and polling succeeds. +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + has_backends +-------------- + t +(1 row) +2: SELECT gpsc.pg_query_state( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1), '\x00112233445566778899aabbccddeeff'::bytea); + pg_query_state +---------------- + +(1 row) + +-- Release the fault and reap the suspended query. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_running_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out new file mode 100644 index 00000000000..b4ad8a9a84d --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/gpsc_pqs_seg_count.out @@ -0,0 +1,58 @@ +-- backends reports exactly one participating backend per primary segment for a +-- single-gang query -- a strict count rather than the has_backends>0 smoke +-- check in gpsc_pqs_running. +-- +-- A plain scan+count is one gang, so the QE list must match the number of +-- primary segments. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_segcount_t (id int) DISTRIBUTED BY (id); +CREATE +INSERT INTO qs_segcount_t SELECT generate_series(1, 100); +INSERT 100 + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +1: SET application_name TO 'qs_segcount_target'; +SET +1&: SELECT count(*) FROM qs_segcount_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- One backend per primary segment. +2: SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role = 'p' AND content > -1) AS matches_primaries FROM gpsc.pg_query_state_backends( (SELECT pid FROM pg_stat_activity WHERE application_name = 'qs_segcount_target' AND pid <> pg_backend_pid() ORDER BY backend_start LIMIT 1)); + matches_primaries +------------------- + t +(1 row) + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) +1<: <... completed> + count +------- + 100 +(1 row) +1q: ... +2q: ... + +DROP TABLE qs_segcount_t; +DROP diff --git a/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out b/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out new file mode 100644 index 00000000000..f7ab4e44725 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/expected/setup.out @@ -0,0 +1,6 @@ +-- Shared setup for the pg_query_state isolation2 suite. +-- pg_isolation2_regress always runs a "setup" test before the schedule. +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; +CREATE diff --git a/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule b/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule new file mode 100644 index 00000000000..5adb0d9cc0a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule @@ -0,0 +1,20 @@ +# pg_query_state isolation2 schedule. +# +# gpsc_pqs_backends -- deterministic: an idle backend yields an empty backend +# list (no fault injector required). +# gpsc_pqs_running -- happy path: a query suspended mid-execution on the QEs +# is observed via pg_query_state_backends/pg_query_state. +# gpsc_pqs_perms -- permission gate: non-superuser non-owner is denied, +# superuser is allowed. +# gpsc_pqs_disabled -- STAT_DISABLED: target with pg_query_state.enable=off +# reports no backends despite a running query. +# gpsc_pqs_seg_count -- strict count: one participating backend per primary +# segment for a single-gang query. +# +# The gpsc_pqs_* specs after gpsc_pqs_backends use gp_inject_fault (enabled by +# default) to suspend a running query while it is polled. +test: gpsc_pqs_backends +test: gpsc_pqs_running +test: gpsc_pqs_perms +test: gpsc_pqs_disabled +test: gpsc_pqs_seg_count diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql new file mode 100644 index 00000000000..4df90380c7a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_backends.sql @@ -0,0 +1,21 @@ +-- pg_query_state_backends against an *idle* backend returns an empty set. +-- +-- Deterministic multi-session check with no async race: session 1 tags itself +-- and sits idle; session 2 looks up its pid and polls it. An idle backend is +-- "not running a query", so GetRemoteBackendInfo returns QUERY_NOT_RUNNING and +-- the function yields an empty set (not an error). +-- +-- Extensions are created by setup.sql. + +-- Session 1: tag connection so session 2 can find its pid, then go idle. +1: SET application_name TO 'qs_idle_target'; +1: SELECT 1; + +-- Session 2: idle target -> zero participating backends. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_idle_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +1q: +2q: diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql new file mode 100644 index 00000000000..10bf81d188a --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_disabled.sql @@ -0,0 +1,36 @@ +-- STAT_DISABLED: when the target backend has pg_query_state.enable = off, its +-- SendCdbComponents reply is STAT_DISABLED, so polling reports an empty backend +-- list even though a query is actively running on the segments. +-- +-- Distinguishes "disabled" from "idle": here the query really is executing +-- (suspended on a fault), yet the disabled module yields nothing. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_disabled_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_disabled_t SELECT generate_series(1, 100); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Target disables the module for its own session, then runs a query that hangs. +1: SET application_name TO 'qs_disabled_target'; +1: SET pg_query_state.enable TO off; +1&: SELECT count(*) FROM qs_disabled_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Running, but module disabled on the target -> empty backend list. +2: SELECT count(*) AS n_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_disabled_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_disabled_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql new file mode 100644 index 00000000000..eadd5c60b33 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_perms.sql @@ -0,0 +1,57 @@ +-- Permission gate: a non-superuser that does not own the target query is +-- denied; a superuser is allowed. +-- +-- The gate is superuser() || GetUserId() == proc->roleId. isolation2 runs all +-- sessions under the same session role (the one that launched the harness), so +-- the "non-super owner is allowed" branch cannot be expressed here and is not +-- covered; the deny and superuser-allow branches are. +-- +-- A non-superuser cannot see another backend's application_name in +-- pg_stat_activity, so the target pid is captured (as superuser) into a table +-- before SET ROLE. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_perm_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_perm_t SELECT generate_series(1, 100); +CREATE TABLE qs_perm_pid (pid int); +CREATE ROLE qs_unpriv; +GRANT SELECT ON qs_perm_pid TO qs_unpriv; +-- No gpsc grants here on purpose: the extension grants USAGE/EXECUTE to PUBLIC +-- in its migration, so an ordinary role reaches the roleId gate exactly as it +-- would in production. This test verifies the gate, not the schema grants. + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +1: SET application_name TO 'qs_perm_target'; +1&: SELECT count(*) FROM qs_perm_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Capture the target pid as superuser (sees application_name). +2: INSERT INTO qs_perm_pid SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_perm_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1; + +-- Non-superuser, non-owner: both entry points are denied. +2: SET ROLE qs_unpriv; +2: SELECT gpsc.pg_query_state((SELECT pid FROM qs_perm_pid), '\x00112233445566778899aabbccddeeff'::bytea); +2: SELECT * FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); +2: RESET ROLE; + +-- Superuser: allowed (non-empty backend list). +2: SELECT count(*) > 0 AS has_backends + FROM gpsc.pg_query_state_backends((SELECT pid FROM qs_perm_pid)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP OWNED BY qs_unpriv; +DROP ROLE qs_unpriv; +DROP TABLE qs_perm_pid; +DROP TABLE qs_perm_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql new file mode 100644 index 00000000000..3c8386cc247 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_running.sql @@ -0,0 +1,45 @@ +-- Happy path: a query suspended mid-execution on the QEs is observed live. +-- +-- Session 1 launches a query that hits an 'executor_pre_tuple_processed' +-- suspend fault on every primary segment, so its QE backends sit inside the +-- executor with a live plan tree. Session 2 then: +-- * pg_query_state_backends(pid) -> at least one participating backend, +-- * pg_query_state(pid) -> succeeds (fire-and-forget, returns void). +-- The fault is reset and the suspended query is reaped. +-- +-- Extensions (gp_stats_collector, gp_inject_fault) come from setup.sql. + +CREATE TABLE qs_running_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_running_t SELECT generate_series(1, 100); + +-- Suspend execution on all primary segments. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Session 1: tag the connection, then launch a query that hangs on the QEs. +1: SET application_name TO 'qs_running_target'; +1&: SELECT count(*) FROM qs_running_t; + +-- Wait until the fault has been hit on the segments. +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- Session 2: the running query has live QE backends, and polling succeeds. +2: SELECT count(*) > 0 AS has_backends FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); +2: SELECT gpsc.pg_query_state( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_running_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1), + '\x00112233445566778899aabbccddeeff'::bytea); + +-- Release the fault and reap the suspended query. +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_running_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql new file mode 100644 index 00000000000..d887f2e71c1 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/gpsc_pqs_seg_count.sql @@ -0,0 +1,36 @@ +-- backends reports exactly one participating backend per primary segment for a +-- single-gang query -- a strict count rather than the has_backends>0 smoke +-- check in gpsc_pqs_running. +-- +-- A plain scan+count is one gang, so the QE list must match the number of +-- primary segments. +-- +-- Extensions come from setup.sql. + +CREATE TABLE qs_segcount_t (id int) DISTRIBUTED BY (id); +INSERT INTO qs_segcount_t SELECT generate_series(1, 100); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'suspend', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +1: SET application_name TO 'qs_segcount_target'; +1&: SELECT count(*) FROM qs_segcount_t; + +SELECT gp_wait_until_triggered_fault('executor_pre_tuple_processed', 1, dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; + +-- One backend per primary segment. +2: SELECT count(*) = (SELECT count(*) FROM gp_segment_configuration + WHERE role = 'p' AND content > -1) AS matches_primaries + FROM gpsc.pg_query_state_backends( + (SELECT pid FROM pg_stat_activity + WHERE application_name = 'qs_segcount_target' AND pid <> pg_backend_pid() + ORDER BY backend_start LIMIT 1)); + +SELECT gp_inject_fault('executor_pre_tuple_processed', 'reset', dbid) + FROM gp_segment_configuration WHERE role = 'p' AND content > -1; +1<: +1q: +2q: + +DROP TABLE qs_segcount_t; diff --git a/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql b/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql new file mode 100644 index 00000000000..faec1135517 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/isolation2/sql/setup.sql @@ -0,0 +1,4 @@ +-- Shared setup for the pg_query_state isolation2 suite. +-- pg_isolation2_regress always runs a "setup" test before the schedule. +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; diff --git a/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql b/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql new file mode 100644 index 00000000000..eb07c45afc2 --- /dev/null +++ b/gpcontrib/gp_stats_collector/test/sql/gpsc_pg_query_state.sql @@ -0,0 +1,46 @@ +-- pg_query_state signal API (extension v1.2): catalog contract + negative paths. +-- +-- Deterministic coverage only: SQL-visible function/type registration and the +-- input-validation error branches. The asynchronous happy path (poll a live +-- query and observe per-node stats) is exercised separately under isolation2, +-- since it depends on a second running backend and timing. +-- start_ignore +CREATE EXTENSION IF NOT EXISTS gp_stats_collector; +-- end_ignore + +-- +-- Catalog contract: the three SQL-visible functions are registered in the gpsc +-- schema with the expected return type and dispatch (exec) location. +-- proexeclocation: c = coordinator, a = any (QE-local), s = all segments. +-- +SELECT proname, + pronargs, + prorettype::regtype AS returns, + proexeclocation +FROM pg_proc +WHERE pronamespace = 'gpsc'::regnamespace + AND proname IN ('pg_query_state', 'pg_query_state_backends', 'cbdb_mpp_query_state') +ORDER BY proname; + +-- Composite identifier type used by the signal layer is present. +SELECT typname +FROM pg_type +WHERE typnamespace = 'gpsc'::regnamespace + AND typname = 'gp_segment_pid'; + +-- +-- Negative: a backend cannot poll its own state. +-- +SELECT gpsc.pg_query_state(pg_backend_pid(), '\x00112233445566778899aabbccddeeff'::bytea); +SELECT * FROM gpsc.pg_query_state_backends(pg_backend_pid()); + +-- +-- Negative: a pid that maps to no live backend is rejected. +-- +SELECT gpsc.pg_query_state(-1, '\x00112233445566778899aabbccddeeff'::bytea); +SELECT * FROM gpsc.pg_query_state_backends(-1); + +-- Cleanup +-- start_ignore +DROP EXTENSION gp_stats_collector; +-- end_ignore diff --git a/pom.xml b/pom.xml index 03ea623c0d7..edc61ad3621 100644 --- a/pom.xml +++ b/pom.xml @@ -1287,6 +1287,9 @@ code or new licensing patterns. gpcontrib/gp_stats_collector/gp_stats_collector.control gpcontrib/gp_stats_collector/.clang-format gpcontrib/gp_stats_collector/Makefile + gpcontrib/gp_stats_collector/test/Makefile + gpcontrib/gp_stats_collector/test/isolation2/Makefile + gpcontrib/gp_stats_collector/test/isolation2/isolation2_schedule gpcontrib/gp_relaccess_stats/Makefile gpcontrib/gp_relaccess_stats/src/gp_relaccess_stats.c diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 0d63d374128..99ff8ef747e 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -1312,15 +1312,37 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) Instrumentation *instr = rInfo->ri_TrigInstrument + nt; char *relname; char *conname = NULL; + instr_time starttimespan; + double total; + double ntuples; + double ncalls; + if (!es->runtime) + { /* Must clean up instrumentation state */ InstrEndLoop(instr); + } + + /* Collect statistic variables */ + if (!INSTR_TIME_IS_ZERO(instr->starttime)) + { + INSTR_TIME_SET_CURRENT(starttimespan); + INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); + } + else + INSTR_TIME_SET_ZERO(starttimespan); + + total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) + + INSTR_TIME_GET_DOUBLE(starttimespan); + ntuples = instr->ntuples + instr->tuplecount; + ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + /* * We ignore triggers that were never invoked; they likely aren't * relevant to the current query type. */ - if (instr->ntuples == 0) + if (ncalls == 0) continue; ExplainOpenGroup("Trigger", NULL, true, es); @@ -1345,10 +1367,10 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) if (show_relname) appendStringInfo(es->str, " on %s", relname); if (es->timing) - appendStringInfo(es->str, ": time=%.3f calls=%.ld\n", - 1000.0 * instr->total, instr->ntuples); + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", + 1000.0 * total, ncalls); else - appendStringInfo(es->str, ": calls=%.ld\n", instr->ntuples); + appendStringInfo(es->str, ": calls=%.0f\n", ncalls); } else { @@ -1357,9 +1379,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) ExplainPropertyText("Constraint Name", conname, es); ExplainPropertyText("Relation", relname, es); if (es->timing) - ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, - es); - ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); + ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); + ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); } if (conname) @@ -2259,8 +2280,11 @@ ExplainNode(PlanState *planstate, List *ancestors, * instrumentation results the user didn't ask for. But we do the * InstrEndLoop call anyway, if possible, to reduce the number of cases * auto_explain has to contend with. + * + * If flag es->stateinfo is set, i.e. when printing the current execution + * state, this step of cleaning up is missed. */ - if (planstate->instrument) + if (planstate->instrument && !es->runtime) InstrEndLoop(planstate->instrument); /* GPDB_90_MERGE_FIXME: In GPDB, these are printed differently. But does that work @@ -2297,7 +2321,7 @@ ExplainNode(PlanState *planstate, List *ancestors, ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); } } - else if (es->analyze) + else if (es->analyze && !es->runtime) { if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoString(es->str, " (never executed)"); @@ -2313,6 +2337,75 @@ ExplainNode(PlanState *planstate, List *ancestors, } } + /* + * Print the progress of node execution at current loop. + */ + if (planstate->instrument && es->analyze && es->runtime) + { + instr_time starttimespan; + double startup_sec; + double total_sec; + double rows; + double loop_num; + bool finished; + + if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) + { + INSTR_TIME_SET_CURRENT(starttimespan); + INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); + } + else + INSTR_TIME_SET_ZERO(starttimespan); + startup_sec = 1000.0 * planstate->instrument->firsttuple; + total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) + + INSTR_TIME_GET_DOUBLE(starttimespan)); + rows = planstate->instrument->tuplecount; + loop_num = planstate->instrument->nloops + 1; + + finished = planstate->instrument->nloops > 0 + && !planstate->instrument->running + && INSTR_TIME_IS_ZERO(starttimespan); + + if (!finished) + { + ExplainOpenGroup("Current loop", "Current loop", true, es); + if (es->format == EXPLAIN_FORMAT_TEXT) + { + if (es->timing) + { + if (planstate->instrument->running) + appendStringInfo(es->str, + " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", + startup_sec, total_sec, rows, loop_num); + else + appendStringInfo(es->str, + " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", + total_sec, loop_num); + } + else + appendStringInfo(es->str, + " (Current loop: actual rows=%.0f, loop number=%.0f)", + rows, loop_num); + } + else + { + ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); + if (es->timing) + { + if (planstate->instrument->running) + { + ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); + ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); + } + else + ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); + } + ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); + } + ExplainCloseGroup("Current loop", "Current loop", true, es); + } + } + /* in text format, first line ends here */ if (es->format == EXPLAIN_FORMAT_TEXT) appendStringInfoChar(es->str, '\n'); @@ -2867,8 +2960,9 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->wal && planstate->instrument) show_wal_usage(es, &planstate->instrument->walusage); - /* Prepare per-worker buffer/WAL usage */ - if (es->workers_state && (es->buffers || es->wal) && es->verbose) + /* Show worker detail after query execution */ + if (es->analyze && es->verbose && planstate->worker_instrument + && !es->runtime) { WorkerInstrumentation *w = planstate->worker_instrument; @@ -4005,6 +4099,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) if (hashstate->hinstrument) memcpy(&hinstrument, hashstate->hinstrument, sizeof(HashInstrumentation)); + + if (hashstate->hashtable) + { + ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); + } /* * Merge results from workers. In the parallel-oblivious case, the @@ -4396,21 +4495,16 @@ show_instrumentation_count(const char *qlabel, int which, if (!es->analyze || !planstate->instrument) return; - + nloops = planstate->instrument->nloops; if (which == 2) - nfiltered = planstate->instrument->nfiltered2; + nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); else - nfiltered = planstate->instrument->nfiltered1; + nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); nloops = planstate->instrument->nloops; /* In text mode, suppress zero counts; they're not interesting enough */ if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) - { - if (nloops > 0) - ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); - else - ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); - } + ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); } /* @@ -5068,15 +5162,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, double insert_path; double other_path; - InstrEndLoop(outerPlanState(mtstate)->instrument); + if (!es->runtime) + InstrEndLoop(outerPlanState(mtstate)->instrument); /* count the number of source rows */ - total = outerPlanState(mtstate)->instrument->ntuples; other_path = mtstate->ps.instrument->ntuples2; - insert_path = total - other_path; - ExplainPropertyFloat("Tuples Inserted", NULL, - insert_path, 0, es); + /* + * Insert occurs after extracting row from subplan and in runtime mode + * we can appear between these two operations - situation when + * total > insert_path + other_path. Therefore we don't know exactly + * whether last row from subplan is inserted. + * We don't print inserted tuples in runtime mode in order to not print + * inconsistent data + */ + if (!es->runtime) + { + total = outerPlanState(mtstate)->instrument->ntuples; + insert_path = total - other_path; + ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); + } + ExplainPropertyFloat("Conflicting Tuples", NULL, other_path, 0, es); } diff --git a/src/backend/executor/instrument.c b/src/backend/executor/instrument.c index 12561e0c051..e85013a4cd5 100644 --- a/src/backend/executor/instrument.c +++ b/src/backend/executor/instrument.c @@ -117,6 +117,9 @@ InstrStopNodeSync(Instrumentation *instr, uint64 nTuples) /* count the returned tuples */ instr->tuplecount += nTuples; + /* A zero-tuple stop means the node is exhausted for this cycle. */ + instr->eof = (nTuples == 0); + /* let's update the time only if the timer was requested */ if (instr->need_timer) { @@ -207,6 +210,7 @@ InstrEndLoop(Instrumentation *instr) /* Reset for next cycle (if any) */ instr->running = false; + instr->eof = false; INSTR_TIME_SET_ZERO(instr->starttime); INSTR_TIME_SET_ZERO(instr->counter); instr->firsttuple = 0; diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 669b5465d73..fc9694bcc14 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -99,12 +99,20 @@ typedef struct #define BARRIER_CLEAR_BIT(flags, type) \ ((flags) &= ~(((uint32) 1) << (uint32) (type))) +#define IsCustomProcSignalReason(reason) \ + ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) + +static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; +static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; +static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; + static ProcSignalHeader *ProcSignal = NULL; static ProcSignalSlot *MyProcSignalSlot = NULL; static bool CheckProcSignal(ProcSignalReason reason); static void CleanupProcSignalState(int status, Datum arg); static void ResetProcSignalBarrierBits(uint32 flags); +static void CheckAndSetCustomSignalInterrupts(void); static bool ProcessBarrierPlaceholder(void); /* @@ -250,6 +258,40 @@ CleanupProcSignalState(int status, Datum arg) slot->pss_pid = 0; } +/* RegisterCustomProcSignalHandler + * Assign specific handler of custom process signal with new + * ProcSignalReason key. + * + * This function has to be called in _PG_init function of extensions at the + * stage of loading shared preloaded libraries. Otherwise it throws fatal error. + * + * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. + */ +ProcSignalReason +RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) +{ + ProcSignalReason reason; + + + if (!process_shared_preload_libraries_in_progress) + { + ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cannot register custom signal after startup"))); + } + + /* Iterate through custom signal slots to find a free one */ + for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) + { + if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) + { + CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; + return reason; + } + } + + return INVALID_PROCSIGNAL; +} + /* * SendProcSignal * Send a signal to a Postgres process @@ -708,7 +750,71 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_FAILED_LOGIN)) HandleLoginFailed(); + CheckAndSetCustomSignalInterrupts(); + SetLatch(MyLatch); errno = save_errno; } + +/* + * Handle receipt of an interrupt indicating any of custom process signals. + */ +static void +CheckAndSetCustomSignalInterrupts() +{ + ProcSignalReason reason; + + for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) + { + if (CheckProcSignal(reason)) + { + /* set interrupt flags */ + InterruptPending = true; + CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; + } + } + + SetLatch(MyLatch); +} + +/* + * CheckAndHandleCustomSignals + * Check custom signal flags and call handler assigned to that signal + * if it is not NULL + * + * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. + */ +void +CheckAndHandleCustomSignals(void) +{ + int i; + + /* + * This is invoked from ProcessInterrupts(), and since some of the + * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential + * for recursive calls if more signals are received while this runs, so + * let's block interrupts until done. + */ + HOLD_INTERRUPTS(); + + /* Check on expiring of custom signals and call its handlers if exist */ + for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) + { + if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) + { + ProcSignalHandler_type handler; + + CustomSignalPendings[i] = false; + handler = CustomInterruptHandlers[i]; + if (handler != NULL) + { + CustomSignalProcessing[i] = true; + handler(); + CustomSignalProcessing[i] = false; + } + } + } + + RESUME_INTERRUPTS(); +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 6ae0202b396..fd1784fa015 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -4363,6 +4363,8 @@ ProcessInterrupts(const char* filename, int lineno) if (ParallelMessagePending) HandleParallelMessages(); + CheckAndHandleCustomSignals(); + if (LogMemoryContextPending) ProcessLogMemoryContextInterrupt(); } @@ -4778,7 +4780,7 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, * postmaster/postmaster.c (the option sets should not conflict) and with * the common help() function in main/main.c. */ - while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lMm:N:nOPp:r:R:S:sTt:v:W:-:")) != -1) + while ((flag = getopt(argc, argv, "B:bc:C:D:d:EeFf:h:ijk:lMm:N:nOPp:r:R:S:sTt:v:W:Z-:")) != -1) { switch (flag) { @@ -4946,6 +4948,10 @@ process_postgres_switches(int argc, char *argv[], GucContext ctx, SetConfigOption("post_auth_delay", optarg, ctx, gucsource); break; + case 'Z': + /* ignored for consistency with the postmaster */ + break; + case 'c': case '-': { diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index 77cb96f0cab..1078460abd4 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -51,6 +51,8 @@ typedef struct ExplainState bool summary; /* print total planning and execution timing */ bool settings; /* print modified settings */ ExplainFormat format; /* output format */ + bool runtime; /* print intermediate state of query execution, + not after completion */ /* state for output formatting --- not reset for each new plan tree */ int indent; /* current indentation level */ List *grouping_stack; /* format-specific grouping state */ diff --git a/src/include/executor/instrument.h b/src/include/executor/instrument.h index 4536df3b237..974c315b46e 100644 --- a/src/include/executor/instrument.h +++ b/src/include/executor/instrument.h @@ -81,6 +81,9 @@ typedef struct Instrumentation bool prf_work; /* true if pushdown runtime filters really work */ /* Info about current plan cycle: */ bool running; /* true if we've completed first tuple */ + bool eof; /* true if the last fetch returned no tuple + * (node exhausted for this cycle); safe to read + * mid-run, unlike nloops/ntuples */ instr_time starttime; /* Start time of current iteration of node */ instr_time counter; /* Accumulated runtime for this node */ double firsttuple; /* Time for first tuple of this cycle */ diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index 0815460c72f..bc9efd6f878 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -15,7 +15,7 @@ #define PROCSIGNAL_H #include "storage/backendid.h" - +#define NUM_CUSTOM_PROCSIGNALS 64 /* * Reasons for signaling a Postgres child process (a backend or an auxiliary @@ -29,6 +29,8 @@ */ typedef enum { + INVALID_PROCSIGNAL = -1, /* Must be first */ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ @@ -49,6 +51,14 @@ typedef enum PROCSIG_FAILED_LOGIN, /* failed login */ + PROCSIG_CUSTOM_1, + /* + * PROCSIG_CUSTOM_2, + * ..., + * PROCSIG_CUSTOM_N-1, + */ + PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, + NUM_PROCSIGNALS /* Must be last! */ } ProcSignalReason; @@ -62,6 +72,9 @@ typedef enum PROCSIGNAL_BARRIER_PLACEHOLDER = 0 } ProcSignalBarrierType; +/* Handler of custom process signal */ +typedef void (*ProcSignalHandler_type) (void); + /* * prototypes for functions in procsignal.c */ @@ -69,12 +82,15 @@ extern Size ProcSignalShmemSize(void); extern void ProcSignalShmemInit(void); extern void ProcSignalInit(int pss_idx); +extern ProcSignalReason +RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); extern int SendProcSignal(pid_t pid, ProcSignalReason reason, BackendId backendId); extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); extern void WaitForProcSignalBarrier(uint64 generation); extern void ProcessProcSignalBarrier(void); +extern void CheckAndHandleCustomSignals(void); extern void procsignal_sigusr1_handler(SIGNAL_ARGS); From 2d5d5156f8adc9aca7260c9760a4c35d9477b4bb Mon Sep 17 00:00:00 2001 From: Vladislav Shchetinin <45269644+Vlasdislav@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:31:18 +0300 Subject: [PATCH 164/167] Update yezzey submodule to tag 1.8.11 (#1945) (#59) --- gpcontrib/yezzey | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpcontrib/yezzey b/gpcontrib/yezzey index 356939e262a..fa0af80fb84 160000 --- a/gpcontrib/yezzey +++ b/gpcontrib/yezzey @@ -1 +1 @@ -Subproject commit 356939e262a03ce6f5fe9c076aaf78c52d499bb9 +Subproject commit fa0af80fb84d9669a5dc342f7bf9df8c4521dbd6 From 96b75285e5aa8998ceafe252dde7b5cc4999f0be Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 26 Aug 2026 13:05:23 +0300 Subject: [PATCH 165/167] Reject direct calls to the gp_percentile transition functions gp_percentile_cont_{float8,interval,timestamp,timestamptz}_transition and gp_percentile_disc_transition all read five arguments in C: the running transition state plus the four arguments of the gp_percentile_cont() and gp_percentile_disc() aggregates. pg_proc.dat, however, declares them with four. As transition functions they are called correctly, since the executor supplies state + 4 arguments regardless of the catalog, but a direct SQL call reaches past the end of the argument array: PG_GETARG_INT64(4) picks up garbage, which yields wrong results, an assertion when the bogus peer count makes the code pfree() a NULL pointer, or a segfault. Correcting the declaration would change the catalog and force an initdb, which is not acceptable on a stable branch, so check the argument count instead and raise a plain error. Direct calls were never useful - the functions only make sense as the transition step of their aggregates - and 'percentile_* WITHIN GROUP' queries are unaffected either way. Co-authored-by: Georgy Shelkovy Ported from Greengage/open-gpdb commit 477b04a (ADBDEV-7770) --- src/backend/utils/adt/orderedsetaggs.c | 22 ++++++++++++++++++++++ src/test/regress/expected/percentile.out | 24 ++++++++++++++++++++++++ src/test/regress/sql/percentile.sql | 14 ++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/src/backend/utils/adt/orderedsetaggs.c b/src/backend/utils/adt/orderedsetaggs.c index 46b2694d89e..efcadadce05 100644 --- a/src/backend/utils/adt/orderedsetaggs.c +++ b/src/backend/utils/adt/orderedsetaggs.c @@ -1516,6 +1516,17 @@ gp_percentile_cont_transition(FunctionCallInfo fcinfo, int64 first_row; int64 second_row; + /* + * Note: 'proargtypes' for this function in pg_proc.dat has 4 arguments. + * There are actually 5 arguments coming in here - the result of the + * previous call and 4 main arguments. + */ + if (PG_NARGS() != 5) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("wrong number of arguments to gp_percentile_cont_transition()"), + errhint("expected 5, got %d", PG_NARGS()))); + /* Return state for NULL inputs of val*/ if (PG_ARGISNULL(1) && !PG_ARGISNULL(0)) PG_RETURN_DATUM(PG_GETARG_DATUM(0)); @@ -1619,6 +1630,17 @@ gp_percentile_disc_transition(PG_FUNCTION_ARGS) { int64 rownum; + /* + * Note: 'proargtypes' for this function in pg_proc.dat has 4 arguments. + * There are actually 5 arguments coming in here - the result of the + * previous call and 4 main arguments. + */ + if (PG_NARGS() != 5) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("wrong number of arguments to gp_percentile_disc_transition()"), + errhint("expected 5, got %d", PG_NARGS()))); + /* Return state for NULL inputs of val*/ if (PG_ARGISNULL(1) && !PG_ARGISNULL(0)) PG_RETURN_DATUM(PG_GETARG_DATUM(0)); diff --git a/src/test/regress/expected/percentile.out b/src/test/regress/expected/percentile.out index 981507d01a3..57d6b2ded5f 100644 --- a/src/test/regress/expected/percentile.out +++ b/src/test/regress/expected/percentile.out @@ -827,6 +827,30 @@ group by d1, d2; 55 | 1 (1 row) +-- +-- gp_percentile_cont()/gp_percentile_disc() are the split ordered-set +-- aggregates that ORCA rewrites percentile_cont()/percentile_disc()/median() +-- into. Their transition functions carry the running state on top of the +-- four aggregate arguments, so they read five arguments, while pg_proc.dat +-- describes four. That is left alone here so as not to force an initdb on a +-- stable branch; instead a direct call, which would read past the end of the +-- argument array, is rejected. +-- +select gp_percentile_cont_float8_transition(NULL::float8, 1, 1, 1); +ERROR: wrong number of arguments to gp_percentile_cont_transition() +HINT: expected 5, got 4 +select gp_percentile_cont_interval_transition(NULL::interval, 1, 1, 1); +ERROR: wrong number of arguments to gp_percentile_cont_transition() +HINT: expected 5, got 4 +select gp_percentile_cont_timestamp_transition(NULL::timestamp, 1, 1, 1); +ERROR: wrong number of arguments to gp_percentile_cont_transition() +HINT: expected 5, got 4 +select gp_percentile_cont_timestamptz_transition(NULL::timestamptz, 1, 1, 1); +ERROR: wrong number of arguments to gp_percentile_cont_transition() +HINT: expected 5, got 4 +select gp_percentile_disc_transition(NULL::numeric, 1, 1, 1); +ERROR: wrong number of arguments to gp_percentile_disc_transition() +HINT: expected 5, got 4 drop view percv2; drop view percv; drop table perct; diff --git a/src/test/regress/sql/percentile.sql b/src/test/regress/sql/percentile.sql index 480adf395b0..cb4930c9609 100644 --- a/src/test/regress/sql/percentile.sql +++ b/src/test/regress/sql/percentile.sql @@ -207,6 +207,20 @@ from mpp_22413 where d2 ='55' group by d1, d2; +-- +-- gp_percentile_cont()/gp_percentile_disc() are the split ordered-set +-- aggregates that ORCA rewrites percentile_cont()/percentile_disc()/median() +-- into. Their transition functions carry the running state on top of the +-- four aggregate arguments, so they read five arguments, while pg_proc.dat +-- describes four. That is left alone here so as not to force an initdb on a +-- stable branch; instead a direct call, which would read past the end of the +-- argument array, is rejected. +-- +select gp_percentile_cont_float8_transition(NULL::float8, 1, 1, 1); +select gp_percentile_cont_interval_transition(NULL::interval, 1, 1, 1); +select gp_percentile_cont_timestamp_transition(NULL::timestamp, 1, 1, 1); +select gp_percentile_cont_timestamptz_transition(NULL::timestamptz, 1, 1, 1); +select gp_percentile_disc_transition(NULL::numeric, 1, 1, 1); drop view percv2; drop view percv; drop table perct; From 84a5bfb302f0cbc54a99139c421436eb53b9a15a Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Wed, 26 Aug 2026 13:06:01 +0300 Subject: [PATCH 166/167] Keep the isnull flag when gp_percentile_* returns its previous state Both gp_percentile transition functions return the previous transition state untouched when the row they are looking at is not one of the rows the percentile is computed from. On the very first call that state is NULL, and returning it as a bare Datum(0) with isnull left false loses the flag: the aggregate yields 0 instead of NULL for by-value types, and dereferences a NULL pointer in the output function for by-reference ones - so an empty input set crashed the backend for the interval, timestamp and timestamptz variants. Co-authored-by: Georgy Shelkovy Ported from Greengage/open-gpdb commit 477b04a (ADBDEV-7770). --- src/backend/utils/adt/orderedsetaggs.c | 21 +++++-- src/test/regress/expected/percentile.out | 70 ++++++++++++++++++++++++ src/test/regress/sql/percentile.sql | 20 +++++++ 3 files changed, 106 insertions(+), 5 deletions(-) diff --git a/src/backend/utils/adt/orderedsetaggs.c b/src/backend/utils/adt/orderedsetaggs.c index efcadadce05..32002dca684 100644 --- a/src/backend/utils/adt/orderedsetaggs.c +++ b/src/backend/utils/adt/orderedsetaggs.c @@ -1573,6 +1573,17 @@ gp_percentile_cont_transition(FunctionCallInfo fcinfo, { return_state = lerpfunc(prev_state, val, proportion); } + else if (PG_ARGISNULL(0)) + { + /* + * Neither of the rows we are after landed in this peer group, so we + * hand the previous state back unchanged. When that state is NULL + * the isnull flag has to travel with it: returning a bare Datum(0) + * as non-NULL gives wrong answers for by-value types and a NULL + * pointer dereference for by-reference ones. + */ + fcinfo->isnull = true; + } *cnt = *cnt + peer_count; if(*cnt > total_rows) @@ -1656,7 +1667,6 @@ gp_percentile_disc_transition(PG_FUNCTION_ARGS) errmsg("percentile value %g is not between 0 and 1", percentile))); Datum prev_state = PG_GETARG_DATUM(0); - bool prev_state_isnull = PG_ARGISNULL(0); Datum val = PG_GETARG_DATUM(1); Datum return_state = prev_state; int64 total_rows = PG_GETARG_INT64(3); @@ -1681,6 +1691,11 @@ gp_percentile_disc_transition(PG_FUNCTION_ARGS) { return_state = val; } + else if (PG_ARGISNULL(0)) + { + /* see gp_percentile_cont_transition() */ + fcinfo->isnull = true; + } *cnt = *cnt + peer_count; @@ -1691,10 +1706,6 @@ gp_percentile_disc_transition(PG_FUNCTION_ARGS) fcinfo->flinfo->fn_extra = NULL; } - if (return_state == prev_state) { - fcinfo->isnull = prev_state_isnull; - } - PG_RETURN_DATUM(return_state); } diff --git a/src/test/regress/expected/percentile.out b/src/test/regress/expected/percentile.out index 57d6b2ded5f..805d8fc13a7 100644 --- a/src/test/regress/expected/percentile.out +++ b/src/test/regress/expected/percentile.out @@ -851,6 +851,76 @@ HINT: expected 5, got 4 select gp_percentile_disc_transition(NULL::numeric, 1, 1, 1); ERROR: wrong number of arguments to gp_percentile_disc_transition() HINT: expected 5, got 4 +-- On an empty input set the transition state stays NULL. The transition +-- functions hand that state back untouched and have to keep its isnull flag +-- with it: a bare Datum(0) escaping here reads as a bogus value for by-value +-- types and dereferences a NULL pointer for by-reference ones. +select gp_percentile_cont(0::float8, 0, 0, 0); + gp_percentile_cont +-------------------- + +(1 row) + +select gp_percentile_cont('0 hour'::interval, 0, 0, 0); + gp_percentile_cont +-------------------- + +(1 row) + +select gp_percentile_cont('2006-01-01 13:10:13'::timestamp, 0, 0, 0); + gp_percentile_cont +-------------------- + +(1 row) + +select gp_percentile_cont('2006-01-01 13:10:13+00'::timestamptz, 0, 0, 0); + gp_percentile_cont +-------------------- + +(1 row) + +select gp_percentile_disc(0::numeric, 0, 0, 0); + gp_percentile_disc +-------------------- + +(1 row) + +-- A value that really was picked has to come back, even when its Datum +-- representation happens to be 0. +select gp_percentile_disc(0::float8, 0, 1, 1); + gp_percentile_disc +-------------------- + 0 +(1 row) + +select gp_percentile_disc(0::int, 0, 1, 1); + gp_percentile_disc +-------------------- + 0 +(1 row) + +select gp_percentile_cont(0::float8, 0, 1, 1); + gp_percentile_cont +-------------------- + 0 +(1 row) + +-- The same, end to end: the smallest value of b is 0. +create table perczero (a int, b float8) distributed by (a); +insert into perczero select i, (i - 1)::float8 from generate_series(1, 10) i; +select percentile_disc(0) within group (order by b) from perczero; + percentile_disc +----------------- + 0 +(1 row) + +select percentile_cont(0) within group (order by b) from perczero; + percentile_cont +----------------- + 0 +(1 row) + +drop table perczero; drop view percv2; drop view percv; drop table perct; diff --git a/src/test/regress/sql/percentile.sql b/src/test/regress/sql/percentile.sql index cb4930c9609..8860df506f7 100644 --- a/src/test/regress/sql/percentile.sql +++ b/src/test/regress/sql/percentile.sql @@ -221,6 +221,26 @@ select gp_percentile_cont_interval_transition(NULL::interval, 1, 1, 1); select gp_percentile_cont_timestamp_transition(NULL::timestamp, 1, 1, 1); select gp_percentile_cont_timestamptz_transition(NULL::timestamptz, 1, 1, 1); select gp_percentile_disc_transition(NULL::numeric, 1, 1, 1); +-- On an empty input set the transition state stays NULL. The transition +-- functions hand that state back untouched and have to keep its isnull flag +-- with it: a bare Datum(0) escaping here reads as a bogus value for by-value +-- types and dereferences a NULL pointer for by-reference ones. +select gp_percentile_cont(0::float8, 0, 0, 0); +select gp_percentile_cont('0 hour'::interval, 0, 0, 0); +select gp_percentile_cont('2006-01-01 13:10:13'::timestamp, 0, 0, 0); +select gp_percentile_cont('2006-01-01 13:10:13+00'::timestamptz, 0, 0, 0); +select gp_percentile_disc(0::numeric, 0, 0, 0); +-- A value that really was picked has to come back, even when its Datum +-- representation happens to be 0. +select gp_percentile_disc(0::float8, 0, 1, 1); +select gp_percentile_disc(0::int, 0, 1, 1); +select gp_percentile_cont(0::float8, 0, 1, 1); +-- The same, end to end: the smallest value of b is 0. +create table perczero (a int, b float8) distributed by (a); +insert into perczero select i, (i - 1)::float8 from generate_series(1, 10) i; +select percentile_disc(0) within group (order by b) from perczero; +select percentile_cont(0) within group (order by b) from perczero; +drop table perczero; drop view percv2; drop view percv; drop table perct; From 5e7f01f88ebe20435629e696b3391ab3562f7957 Mon Sep 17 00:00:00 2001 From: Alena Rybakina Date: Tue, 25 Aug 2026 18:29:47 +0300 Subject: [PATCH 167/167] Keep no-match rows when pulling up a correlated aggregate subquery With the Postgres planner (optimizer=off or an ORCA fallback), a correlated scalar subquery with an aggregate is pulled up into an INNER join with a grouped subquery (convert_EXPR_to_join), which drops outer rows that have no match. The original subquery keeps them: it computes the aggregate over empty input, so e.g. COUNT yields 0 there: select ... from t1 where t1.a > (select count(*) from t2 where t2.a = t1.d); A row with no match in t2 must be compared as "t1.a > 0" and can pass, but the INNER join dropped it. To fix this, pull the subquery up into a LEFT join, so no-match rows survive as null-extended rows, and rewrite the comparison to return the same value the subquery would: outer OP CASE WHEN match_flag THEN expr ELSE empty_input_default END match_flag is a constant TRUE column added to the subquery. For a matched row the CASE returns the real expression; for a null-extended row the flag is NULL and the CASE returns the empty-input default (0 for COUNT, NULL for other aggregates). The comparison runs above the LEFT join as a filter, not as the join condition: as a join qual it would null-extend matched rows that fail it, and the default would let them back in. The LEFT join is not always needed. If a no-match row cannot pass the comparison anyway -- e.g. "1 = (select count(*) ...)" turns into "1 = 0" for it -- dropping it is fine and the INNER join is kept as before. This is detected by substituting the empty-input default into the comparison and constant-folding it. Ordinary sum/avg/min/max comparisons fall into this group: their empty-input value is NULL, and a comparison with NULL does not pass, so those plans do not change. If the comparison cannot be placed above the join (the sublink is in an outer join's ON clause) or the subquery's targetlist is correlated, the pull-up bails out and the sublink runs as a SubPlan, as before. Adapted from open-gpdb https://github.com/open-gpdb/gpdb/pull/397 and Greengage https://github.com/GreengageDB/greengage/pull/546. Co-Authored-By: excaliiibur [excaliiibur@foxmail.com] (cherry picked from commit ffdcd7818fd3d2fba2405952962880d2adbf346a) --- .../src/test/regress/expected/bfv_dd.out | 1 + .../src/test/regress/expected/eagerfree.out | 26 +- src/backend/cdb/cdbsubselect.c | 244 +++++++++++++++++- src/backend/optimizer/prep/prepjointree.c | 28 ++ src/test/regress/expected/bfv_dd.out | 1 + src/test/regress/expected/eagerfree.out | 26 +- 6 files changed, 296 insertions(+), 30 deletions(-) diff --git a/contrib/pax_storage/src/test/regress/expected/bfv_dd.out b/contrib/pax_storage/src/test/regress/expected/bfv_dd.out index 46b9f092f5e..86439d02547 100644 --- a/contrib/pax_storage/src/test/regress/expected/bfv_dd.out +++ b/contrib/pax_storage/src/test/regress/expected/bfv_dd.out @@ -888,6 +888,7 @@ INFO: (slice 2) Dispatch command to SINGLE content -- subqueries select * from dd_singlecol_1 t1 where a=1 and b < (select count(*) from dd_singlecol_2 t2 where t2.a=t1.a); +INFO: (slice 3) Dispatch command to ALL contents: 0 1 2 INFO: (slice 2) Dispatch command to ALL contents: 0 1 2 INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 a | b diff --git a/contrib/pax_storage/src/test/regress/expected/eagerfree.out b/contrib/pax_storage/src/test/regress/expected/eagerfree.out index 53be6095c14..1df19af37f0 100644 --- a/contrib/pax_storage/src/test/regress/expected/eagerfree.out +++ b/contrib/pax_storage/src/test/regress/expected/eagerfree.out @@ -1379,21 +1379,19 @@ where i < (select count(*) from smallt where smallt.i = smallt2.i) order by 1,2, explain select smallt2.* from smallt2 where i < (select count(*) from smallt where smallt.i = smallt2.i); - QUERY PLAN ---------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=5.10..8.08 rows=17 width=15) - -> Hash Join (cost=5.10..8.08 rows=6 width=15) - Hash Cond: smallt2.i = "Expr_SUBQUERY".csq_c0 - Join Filter: smallt2.i < "Expr_SUBQUERY".csq_c1 - -> Seq Scan on smallt2 (cost=0.00..2.50 rows=17 width=15) - -> Hash (cost=4.97..4.97 rows=4 width=12) - -> Subquery Scan on "Expr_SUBQUERY" (cost=4.75..4.97 rows=4 width=12) - -> HashAggregate (cost=4.75..4.88 rows=4 width=12) - Filter: smallt.i < count(*) - Group Key: smallt.i - -> Seq Scan on smallt (cost=0.00..4.00 rows=34 width=4) + QUERY PLAN +------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.61..3.99 rows=17 width=15) + -> Hash Left Join (cost=1.61..2.88 rows=6 width=15) + Hash Cond: (smallt2.i = smallt.i) + Filter: (smallt2.i < CASE WHEN (true) THEN (count(*)) ELSE '0'::bigint END) + -> Seq Scan on smallt2 (cost=0.00..1.17 rows=17 width=15) + -> Hash (cost=1.57..1.57 rows=3 width=13) + -> HashAggregate (cost=1.50..1.53 rows=3 width=13) + Group Key: smallt.i + -> Seq Scan on smallt (cost=0.00..1.33 rows=33 width=4) Optimizer: Postgres query optimizer -(12 rows) +(10 rows) -- Sort in MergeJoin -- start_ignore diff --git a/src/backend/cdb/cdbsubselect.c b/src/backend/cdb/cdbsubselect.c index 43a46eace4b..d3efa37a4e0 100644 --- a/src/backend/cdb/cdbsubselect.c +++ b/src/backend/cdb/cdbsubselect.c @@ -29,6 +29,7 @@ #include "parser/parse_relation.h" /* addRangeTableEntryForSubquery() */ #include "parser/parsetree.h" /* rt_fetch() */ #include "rewrite/rewriteManip.h" +#include "utils/fmgroids.h" /* F_COUNT_ANY, F_COUNT_ */ #include "utils/lsyscache.h" /* get_op_btree_interpretation() */ #include "utils/syscache.h" #include "cdb/cdbsubselect.h" /* me */ @@ -42,6 +43,21 @@ static JoinExpr *make_join_expr(Node *larg, int r_rtindex, int join_type); static Node *make_lasj_quals(PlannerInfo *root, SubLink *sublink, int subquery_indx); static Node *add_null_match_clause(Node *clause); +static Expr *build_match_flag_case_expr(Var *flagVar, Var *aggVar, Expr *defaultExpr); + +/* + * State of replace_agg_with_empty_default_mutator(). + */ +typedef struct EmptyInputDefaultContext +{ + bool sawNonNullDefault; /* has a COUNT been replaced by 0? */ +} EmptyInputDefaultContext; + +static Expr *build_empty_input_default_expr(Node *expr, + EmptyInputDefaultContext *ctx); +static Node *replace_agg_with_empty_default_mutator(Node *node, void *context); +static bool no_match_row_survives(PlannerInfo *root, OpExpr *opexp, + Expr *defaultExpr); typedef struct NonNullableVarsContext { @@ -538,6 +554,18 @@ safe_to_convert_EXPR(SubLink *sublink, ConvertSubqueryToJoinContext *ctx1) if (!subselect->hasAggs) return false; + /* + * A window function cannot survive the pull-up. Without it the subquery + * has a plain aggregate and so produces exactly one row per outer row, and + * the window runs over that single row. The pulled-up subquery is grouped + * by the correlation columns, so the same window would run over every + * group at once and compute a different value. (The rewrite of the + * comparison below would also copy the WindowFunc into a qual above the + * join, where there is no WindowAgg node to evaluate it.) + */ + if (subselect->hasWindowFuncs) + return false; + /** * A LIMIT or OFFSET could interfere with the transformation of the * correlated qual to GROUP BY. (LIMIT >0 in a subquery that contains a @@ -560,6 +588,14 @@ safe_to_convert_EXPR(SubLink *sublink, ConvertSubqueryToJoinContext *ctx1) if (list_length(subselect->targetList) != 1) return false; + /** + * Correlation in the targetlist cannot be handled: the pulled-up + * expression (and the empty-input default derived from it) would carry + * upper-level Vars out of the subquery. + */ + if (contain_vars_of_level_or_above((Node *) subselect->targetList, 1)) + return false; + /** * Walk the quals of the subquery to do a more fine grained check as to whether this subquery @@ -623,6 +659,72 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) subselect->jointree->quals = ctx1.innerQual; + /* + * An INNER join drops outer rows that have no matching inner + * rows. Without the pull-up they are kept: the subquery + * computes its expression over empty input (COUNT = 0, other + * aggregates NULL) and the comparison may still pass. + * + * So plug the empty-input value into the comparison and run + * eval_const_expressions() on it. FALSE or NULL means no-match + * rows cannot pass and the INNER join is correct; otherwise use + * a LEFT join to keep them. + */ + Expr *defaultExpr; + TargetEntry *flagTLE = NULL; + bool use_left_join; + EmptyInputDefaultContext defaultCtx; + + defaultExpr = build_empty_input_default_expr((Node *) origSubqueryTLE->expr, + &defaultCtx); + + /* + * defaultExpr ends up in a qual of the outer query, where the planner + * folds constant expressions at plan time. Substituting 0 for a COUNT + * can turn a subexpression the original query only ever evaluated per + * row into a constant one: "1/count(*)" becomes "1/0" and raises + * "division by zero" while planning, even for a query that returns no + * rows at all. A NULL default cannot do that -- it only propagates + * through strict functions, which the planner folds without calling + * them, and anything it does evaluate was already constant in the + * original expression. + * + * So take the LEFT-join path only when the invented value is a plain + * constant; otherwise leave the sublink to be planned as a SubPlan, + * which keeps the original semantics. + */ + if (defaultCtx.sawNonNullDefault && !IsA(defaultExpr, Const)) + return NULL; + + use_left_join = no_match_row_survives(root, opexp, defaultExpr); + + if (use_left_join) + { + /* + * After the LEFT join the expression column is NULL both for a + * no-match row and for a matched group whose expression is + * genuinely NULL. To tell them apart, add a constant-TRUE + * match-flag column to the subquery: it can be NULL only when + * the LEFT join found no match and filled the subquery's + * columns with NULLs. + * + * The flag goes BEFORE the expression column: with this + * order the planner can drop the SubqueryScan node from the + * plan. + */ + TargetEntry *aggTLE = (TargetEntry *) llast(subselect->targetList); + + flagTLE = makeTargetEntry((Expr *) makeBoolConst(true, false), + aggTLE->resno, + pstrdup("csq_count_flag"), + false); + aggTLE->resno++; + subselect->targetList = list_truncate(subselect->targetList, + list_length(subselect->targetList) - 1); + subselect->targetList = lappend(subselect->targetList, flagTLE); + subselect->targetList = lappend(subselect->targetList, aggTLE); + } + /** * Construct a new range table entry for the new pulled up subquery. */ @@ -644,7 +746,8 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) join_expr->quals = joinQual; - TargetEntry *subselectAggTLE = (TargetEntry *) list_nth(subselect->targetList, list_length(subselect->targetList) - 1); + /* The pulled-up expression column is last in either layout. */ + TargetEntry *subselectAggTLE = (TargetEntry *) llast(subselect->targetList); /** * modify the op expr to involve the column that has the computed aggregate that needs to compared. @@ -656,7 +759,20 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) exprCollation((Node *) subselectAggTLE->expr), 0); - list_nth_replace(opexp->args, 1, aggVar); + if (use_left_join) + { + Var *flagVar; + + join_expr->jointype = JOIN_LEFT; + flagVar = (Var *) makeVar(rteIndex, flagTLE->resno, BOOLOID, -1, + InvalidOid, 0); + list_nth_replace(opexp->args, 1, + build_match_flag_case_expr(flagVar, aggVar, defaultExpr)); + } + else + { + list_nth_replace(opexp->args, 1, aggVar); + } return join_expr; } @@ -664,6 +780,130 @@ convert_EXPR_to_join(PlannerInfo *root, OpExpr *opexp) return NULL; } +/* + * Build "CASE WHEN flagVar THEN aggVar ELSE defaultExpr END". + * + * flagVar is the subquery's match-flag column: TRUE for a matched group, + * NULL for a null-extended no-match row. + */ +static Expr * +build_match_flag_case_expr(Var *flagVar, Var *aggVar, Expr *defaultExpr) +{ + CaseWhen *casewhen; + CaseExpr *caseexpr; + + Assert(flagVar != NULL); + Assert(aggVar != NULL); + Assert(defaultExpr != NULL); + + casewhen = makeNode(CaseWhen); + casewhen->expr = (Expr *) flagVar; + casewhen->result = (Expr *) aggVar; + casewhen->location = -1; + + caseexpr = makeNode(CaseExpr); + caseexpr->casetype = exprType((Node *) aggVar); + caseexpr->casecollid = exprCollation((Node *) aggVar); + caseexpr->arg = NULL; + caseexpr->args = list_make1(casewhen); + caseexpr->defresult = defaultExpr; + caseexpr->location = -1; + + return (Expr *) caseexpr; +} + +/* + * Build the value the subquery's expression takes over empty input, by + * replacing every aggregate with its own empty-input value. + * + * ctx->sawNonNullDefault reports whether the result rests on a value this + * code invented, rather than on a NULL the original expression would have + * produced anyway. Only COUNT does that; the caller uses it to decide + * whether the result is safe to plant in a qual of the outer query. + */ +static Expr * +build_empty_input_default_expr(Node *expr, EmptyInputDefaultContext *ctx) +{ + ctx->sawNonNullDefault = false; + + return (Expr *) replace_agg_with_empty_default_mutator(copyObject(expr), + ctx); +} + +static Node * +replace_agg_with_empty_default_mutator(Node *node, void *context) +{ + EmptyInputDefaultContext *ctx = (EmptyInputDefaultContext *) context; + Aggref *aggref; + Oid default_type; + Oid default_collation; + int16 typlen; + bool typbyval; + + if (node == NULL) + return NULL; + + if (IsA(node, Aggref)) + { + bool is_count; + + aggref = (Aggref *) node; + is_count = (aggref->aggfnoid == F_COUNT_ANY || + aggref->aggfnoid == F_COUNT_); + if (is_count) + { + default_type = INT8OID; + default_collation = InvalidOid; + } + else + { + default_type = aggref->aggtype; + default_collation = exprCollation((Node *) aggref); + } + + /* + * COUNT is 0 over empty input; every other aggregate is NULL. The + * choice must follow the aggregate, not its result type: sum(int4) + * also returns int8 but its empty-input value is NULL. + */ + get_typlenbyval(default_type, &typlen, &typbyval); + if (is_count) + ctx->sawNonNullDefault = true; + return (Node *) makeConst(default_type, -1, default_collation, typlen, + is_count ? Int64GetDatum(0) : (Datum) 0, + !is_count, typbyval); + } + + return expression_tree_mutator(node, replace_agg_with_empty_default_mutator, + context); +} + +/* + * no_match_row_survives + * + * Could a no-match row satisfy "outerExpr OP (subquery)"? Plug defaultExpr in + * for the subquery and constant-fold: false if it folds to FALSE/NULL, else true. + */ +static bool +no_match_row_survives(PlannerInfo *root, OpExpr *opexp, Expr *defaultExpr) +{ + OpExpr *testexpr = (OpExpr *) copyObject(opexp); + Node *folded; + + list_nth_replace(testexpr->args, 1, copyObject(defaultExpr)); + folded = eval_const_expressions(root, (Node *) testexpr); + + if (IsA(folded, Const)) + { + Const *c = (Const *) folded; + + if (c->constisnull || !DatumGetBool(c->constvalue)) + return false; + } + + return true; +} + /* NOTIN subquery transformation -start */ /* check if NOT IN conversion to antijoin is possible */ diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 3b9b4f39bb0..fe6402e1d34 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -758,11 +758,39 @@ pull_up_sublinks_qual_recurse(PlannerInfo *root, Node *node, if (IsA(rarg, SubLink)) { + /* + * The pulled-up join is spliced in at *jtlink1, and in the + * LEFT-join case the comparison itself moves there too, so + * every Var of this query level used by the clause must be + * available at that attach point. Otherwise (e.g. an outer + * join's ON clause referencing the non-nullable side) leave + * the sublink to be planned as a SubPlan. + */ + if (!bms_is_subset(pull_varnos(root, node), available_rels1)) + return node; + j = convert_EXPR_to_join(root, opexp); if (j) { /* Yes, insert the new join node into the join tree */ j->larg = *jtlink1; + + if (j->jointype == JOIN_LEFT) + { + /* + * COUNT-preserving pull-up (see convert_EXPR_to_join). + * opexp must run ABOVE the LEFT JOIN, not as its join + * condition: as a join qual a matched row that fails it + * would be treated as unmatched, null-extended, and let + * back in by the no-match default of the CASE built by + * convert_EXPR_to_join. Wrap the join in a FromExpr so + * opexp stays a post-join filter. + */ + *jtlink1 = (Node *) makeFromExpr(list_make1(j), node); + return NULL; + } + + /* Inner-join case: opexp stays as an ordinary qual. */ *jtlink1 = (Node *) j; } return node; diff --git a/src/test/regress/expected/bfv_dd.out b/src/test/regress/expected/bfv_dd.out index 46b9f092f5e..86439d02547 100644 --- a/src/test/regress/expected/bfv_dd.out +++ b/src/test/regress/expected/bfv_dd.out @@ -888,6 +888,7 @@ INFO: (slice 2) Dispatch command to SINGLE content -- subqueries select * from dd_singlecol_1 t1 where a=1 and b < (select count(*) from dd_singlecol_2 t2 where t2.a=t1.a); +INFO: (slice 3) Dispatch command to ALL contents: 0 1 2 INFO: (slice 2) Dispatch command to ALL contents: 0 1 2 INFO: (slice 1) Dispatch command to ALL contents: 0 1 2 a | b diff --git a/src/test/regress/expected/eagerfree.out b/src/test/regress/expected/eagerfree.out index 9658e9a8dd6..c6de8531a51 100644 --- a/src/test/regress/expected/eagerfree.out +++ b/src/test/regress/expected/eagerfree.out @@ -1379,21 +1379,19 @@ where i < (select count(*) from smallt where smallt.i = smallt2.i) order by 1,2, explain select smallt2.* from smallt2 where i < (select count(*) from smallt where smallt.i = smallt2.i); - QUERY PLAN ---------------------------------------------------------------------------------------- - Gather Motion 3:1 (slice1; segments: 3) (cost=5.10..8.08 rows=17 width=15) - -> Hash Join (cost=5.10..8.08 rows=6 width=15) - Hash Cond: smallt2.i = "Expr_SUBQUERY".csq_c0 - Join Filter: smallt2.i < "Expr_SUBQUERY".csq_c1 - -> Seq Scan on smallt2 (cost=0.00..2.50 rows=17 width=15) - -> Hash (cost=4.97..4.97 rows=4 width=12) - -> Subquery Scan on "Expr_SUBQUERY" (cost=4.75..4.97 rows=4 width=12) - -> HashAggregate (cost=4.75..4.88 rows=4 width=12) - Filter: smallt.i < count(*) - Group Key: smallt.i - -> Seq Scan on smallt (cost=0.00..4.00 rows=34 width=4) + QUERY PLAN +------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.61..3.99 rows=17 width=15) + -> Hash Left Join (cost=1.61..2.88 rows=6 width=15) + Hash Cond: (smallt2.i = smallt.i) + Filter: (smallt2.i < CASE WHEN (true) THEN (count(*)) ELSE '0'::bigint END) + -> Seq Scan on smallt2 (cost=0.00..1.17 rows=17 width=15) + -> Hash (cost=1.57..1.57 rows=3 width=13) + -> HashAggregate (cost=1.50..1.53 rows=3 width=13) + Group Key: smallt.i + -> Seq Scan on smallt (cost=0.00..1.33 rows=33 width=4) Optimizer: Postgres query optimizer -(12 rows) +(10 rows) -- Sort in MergeJoin -- start_ignore