Skip to content

Several fixes to storage settings` migration #70

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Smyatkin-Maxim wants to merge 1 commit into
base: REL_2_STABLE
Choose a base branch
from
Open

Several fixes to storage settings` migration #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 148 additions & 1 deletion src/bin/pg_dump/dumputils.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include <ctype.h>

#include "common/logging.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"

Expand Down Expand Up @@ -888,6 +889,138 @@ SplitGUCList(char *rawstring, char separator,
return true;
}

static void makeAlterConfigCommandInternal(PGconn *conn, const char *configitem,
const char *type, const char *name,
const char *type2, const char *name2,
PQExpBuffer buf);

static char *
trimStorageWhitespace(char *s)
{
char *end;

while (isspace((unsigned char) *s))
s++;
end = s + strlen(s);
while (end > s && isspace((unsigned char) end[-1]))
*--end = '\0';
return s;
}

/* Accept the same unambiguous boolean prefixes as the source server. */
static bool
parseStorageBool(const char *value, bool *result)
{
size_t len = strlen(value);

if (len == 0)
return false;
if (pg_strncasecmp(value, "true", len) == 0 ||
pg_strncasecmp(value, "yes", len) == 0 ||
(len >= 2 && pg_strncasecmp(value, "on", len) == 0) ||
strcmp(value, "1") == 0)
*result = true;
else if (pg_strncasecmp(value, "false", len) == 0 ||
pg_strncasecmp(value, "no", len) == 0 ||
(len >= 2 && pg_strncasecmp(value, "off", len) == 0) ||
strcmp(value, "0") == 0)
*result = false;
else
return false;
return true;
}

/* Split legacy storage defaults at the same database/role scope. */
static bool
dumpGpDefaultStorageOptions(PGconn *conn, const char *guc,
const char *type, const char *name,
const char *type2, const char *name2,
PQExpBuffer outbuf)
{
static const char prefix[] = "gp_default_storage_options=";
char *value;
char *tok;
char *next;
int ao_state = -1;
bool orient_seen = false;
bool is_column = false;
const char *am;
PQExpBuffer remaining;
PQExpBuffer opt;

if (strncmp(guc, prefix, strlen(prefix)) != 0)
return false;

remaining = createPQExpBuffer();
value = pg_strdup(guc + strlen(prefix));

for (tok = *value ? value : NULL; tok != NULL; tok = next)
{
char *optname;
char *val;
char *eq;
bool ao;

next = strchr(tok, ',');
if (next)
*next++ = '\0';
eq = strchr(tok, '=');
if (eq == NULL)
goto invalid;
*eq = '\0';
optname = trimStorageWhitespace(tok);
val = trimStorageWhitespace(eq + 1);
if (*optname == '\0' || *val == '\0' || strchr(val, '='))
goto invalid;

if (pg_strcasecmp(optname, "appendonly") == 0 ||
pg_strcasecmp(optname, "appendoptimized") == 0)
{
if (ao_state != -1 || !parseStorageBool(val, &ao))
goto invalid;
ao_state = ao ? 1 : 0;
}
else if (pg_strcasecmp(optname, "orientation") == 0)
{
if (orient_seen ||
(pg_strcasecmp(val, "column") != 0 &&
pg_strcasecmp(val, "row") != 0))
goto invalid;
orient_seen = true;
is_column = (pg_strcasecmp(val, "column") == 0);
}
else
{
if (remaining->len > 0)
appendPQExpBufferChar(remaining, ',');
appendPQExpBuffer(remaining, "%s=%s", optname, val);
}
}

if (ao_state != 1)
am = "default_table_access_method=heap";
else if (is_column)
am = "default_table_access_method=ao_column";
else
am = "default_table_access_method=ao_row";

/* Both settings must override lower-priority scopes, even when empty. */
makeAlterConfigCommandInternal(conn, am, type, name, type2, name2, outbuf);
opt = createPQExpBuffer();
appendPQExpBuffer(opt, "gp_default_storage_options=%s", remaining->data);
makeAlterConfigCommandInternal(conn, opt->data, type, name, type2, name2,
outbuf);
destroyPQExpBuffer(opt);
destroyPQExpBuffer(remaining);
pg_free(value);
return true;

invalid:
pg_log_error("invalid gp_default_storage_options setting for %s \"%s\": \"%s\"",
type, name, guc);
exit(EXIT_FAILURE);
}

/*
* Helper function for dumping "ALTER DATABASE/ROLE SET ..." commands.
*
Expand All @@ -897,13 +1030,27 @@ SplitGUCList(char *rawstring, char separator,
* type is DATABASE or ROLE, and name is the name of the database or role.
* If we need an "IN" clause, type2 and name2 similarly define what to put
* there; otherwise they should be NULL.
* conn is used only to determine string-literal quoting conventions.
* conn supplies the source version and string-literal quoting conventions.
*/
void
makeAlterConfigCommand(PGconn *conn, const char *configitem,
const char *type, const char *name,
const char *type2, const char *name2,
PQExpBuffer buf)
{
/* GP7+ stores the access method in a separate GUC already. */
if (PQserverVersion(conn) < 120000 &&
dumpGpDefaultStorageOptions(conn, configitem, type, name, type2, name2, buf))
return;

makeAlterConfigCommandInternal(conn, configitem, type, name, type2, name2, buf);
}

static void
makeAlterConfigCommandInternal(PGconn *conn, const char *configitem,
const char *type, const char *name,
const char *type2, const char *name2,
PQExpBuffer buf)
{
char *mine;
char *pos;
Expand Down
73 changes: 42 additions & 31 deletions src/bin/pg_dump/pg_dump.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7202,14 +7202,19 @@ getTables(Archive *fout, int *numTables)
appendPQExpBufferStr(query,
"0 AS relminmxid, 0 AS tminmxid, ");

/* Storage-model reloptions are emitted from the table access method. */
if (fout->remoteVersion >= 90300)
appendPQExpBufferStr(query,
"array_remove(array_remove(c.reloptions,'check_option=local'),'check_option=cascaded') AS reloptions, "
"ARRAY(SELECT x FROM unnest("
"array_remove(array_remove(c.reloptions,'check_option=local'),'check_option=cascaded')"
") x WHERE x NOT LIKE 'appendonly=%' AND x NOT LIKE 'orientation=%') AS reloptions, "
"CASE WHEN 'check_option=local' = ANY (c.reloptions) THEN 'LOCAL'::text "
"WHEN 'check_option=cascaded' = ANY (c.reloptions) THEN 'CASCADED'::text ELSE NULL END AS checkoption, ");
else
appendPQExpBufferStr(query,
"c.reloptions, NULL AS checkoption, ");
"ARRAY(SELECT x FROM unnest(c.reloptions) x "
"WHERE x NOT LIKE 'appendonly=%' AND x NOT LIKE 'orientation=%') AS reloptions, "
"NULL AS checkoption, ");

if (fout->remoteVersion >= 80400)
appendPQExpBufferStr(query,
Expand All @@ -7229,9 +7234,21 @@ getTables(Archive *fout, int *numTables)
appendPQExpBufferStr(query,
"am.amname, am.oid as amoid, ");
else
appendPQExpBufferStr(query,
"NULL AS amname, NULL as amoid, ");

/* Explicit source AMs prevent target defaults from changing table storage. */
appendPQExpBuffer(query,
"CASE c.relstorage "
"WHEN 'a' THEN 'ao_row' "
"WHEN 'c' THEN 'ao_column' "
"WHEN 'h' THEN 'heap' "
"ELSE NULL END AS amname, "
"CASE c.relstorage "
"WHEN 'a' THEN %u::oid "
"WHEN 'c' THEN %u::oid "
"WHEN 'h' THEN %u::oid "
"ELSE NULL END AS amoid, ",
AO_ROW_TABLE_AM_OID,
AO_COLUMN_TABLE_AM_OID,
HEAP_TABLE_AM_OID);
if (fout->remoteVersion >= 90600)
appendPQExpBufferStr(query,
"c.relkind = " CppAsString2(RELKIND_SEQUENCE)
Expand Down Expand Up @@ -17834,11 +17851,9 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
*/
appendPQExpBufferStr(q, " INTEGER /* dummy */");

/* Dropped columns are dumped during binary upgrade.
* Dump the encoding clause also to maintain a consistent
* catalog entry in pg_attribute_encoding post upgrade.
*/
if (tbinfo->attencoding[j] != NULL)
/* Preserve dropped-column encodings only for AOCO tables. */
if (tbinfo->amoid == AO_COLUMN_TABLE_AM_OID &&
tbinfo->attencoding[j] != NULL)
appendPQExpBuffer(q, " ENCODING (%s)", tbinfo->attencoding[j]);

/* Skip all the rest */
Expand Down Expand Up @@ -17911,30 +17926,26 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
actual_atts++;
}

/*
* Add AOCO ENCODING directives, if any.
*
* We dump these as separate "COLUMN <col> ENCODING ..." clauses,
* instead of tacking the ENCODING at the column definition, so
* that this works for inherited columns, too. Inherited columns
* are not listed in the column list.
*/
for (j = 0; j < tbinfo->numatts; j++)
/* Separate AOCO ENCODING clauses also cover inherited columns. */
if (tbinfo->amoid == AO_COLUMN_TABLE_AM_OID)
{
if (tbinfo->attisdropped[j])
continue;

if (tbinfo->attencoding[j] != NULL)
for (j = 0; j < tbinfo->numatts; j++)
{
if (actual_atts == 0)
appendPQExpBufferStr(q, " (\n ");
else
appendPQExpBufferStr(q, ",\n ");
if (tbinfo->attisdropped[j])
continue;

appendPQExpBuffer(q, "COLUMN %s ENCODING (%s)",
fmtId(tbinfo->attnames[j]),
tbinfo->attencoding[j]);
actual_atts++;
if (tbinfo->attencoding[j] != NULL)
{
if (actual_atts == 0)
appendPQExpBufferStr(q, " (\n ");
else
appendPQExpBufferStr(q, ",\n ");

appendPQExpBuffer(q, "COLUMN %s ENCODING (%s)",
fmtId(tbinfo->attnames[j]),
tbinfo->attencoding[j]);
actual_atts++;
}
}
}

Expand Down
Loading