[Backups] Fix three failure reason classifications

pkg/pgbackrest/command.go

@@ -92,10 +92,11 @@ func IsLockBusy(err error) bool {
 }
 
 const (
-	protocolExitCode    = 39
-	hostConnectExitCode = 49
-	fileMissingExitCode = 55
-	dbConnectExitCode   = 56
+	protocolExitCode       = 39
+	hostConnectExitCode    = 49
+	fileMissingExitCode    = 55
+	dbConnectExitCode      = 56
+	archiveTimeoutExitCode = 82
 )
 
 // ClassifyExitCode returns the failure reason for a pgbackrest CommandError, or false for any other error.
@@ -105,17 +106,26 @@ func ClassifyExitCode(err error) (apiv1.BackupFailureReason, bool) {
 		return "", false
 	}
 
+	// The handshake error is not tied to one exit code. On a replica, a failed
+	// handshake with the remote primary is only a warning, and pgbackrest then
+	// exits 56 because it found no primary.
+	if isVersionMismatch(cmdErr.Stderr) {
+		return apiv1.BackupFailureReasonVersionMismatch, true
+	}
+
 	switch cmdErr.ExitCode {
 	case lockAcquireExitCode:
 		return apiv1.BackupFailureReasonLockContention, true
 	case dbConnectExitCode:
 		return apiv1.BackupFailureReasonDBUnavailable, true
 	case hostConnectExitCode:
-		return apiv1.BackupFailureReasonStandbyUnreachable, true
+		return classifyHostConnectError(cmdErr.Stderr), true
 	case fileMissingExitCode:
 		return apiv1.BackupFailureReasonStanzaNotReady, true
 	case protocolExitCode:
 		return classifyProtocolError(cmdErr.Stderr), true
+	case archiveTimeoutExitCode:
+		return classifyArchiveTimeoutError(cmdErr.Stderr), true
 	default:
 		return apiv1.BackupFailureReasonPgBackRestError, true
 	}
@@ -127,8 +137,6 @@ func classifyProtocolError(stderr string) apiv1.BackupFailureReason {
 		strings.Contains(stderr, "Forbidden") ||
 		strings.Contains(stderr, "AccessDenied"):
 		return apiv1.BackupFailureReasonObjectStoreDenied
-	case strings.Contains(stderr, "greeting key 'version'"):
-		return apiv1.BackupFailureReasonVersionMismatch
 	case strings.Contains(stderr, fmt.Sprintf(":%d", TLSServerPort)) ||
 		strings.Contains(strings.ToLower(stderr), "tls"):
 		return apiv1.BackupFailureReasonStandbyUnreachable
@@ -137,6 +145,31 @@ func classifyProtocolError(stderr string) apiv1.BackupFailureReason {
 	}
 }
 
+// classifyHostConnectError separates a pg host from the object store. A pg
+// host listens on TLSServerPort, and an error from a remote pgbackrest carries
+// the "raised from remote" prefix. Any other host is the object store.
+func classifyHostConnectError(stderr string) apiv1.BackupFailureReason {
+	if strings.Contains(stderr, fmt.Sprintf(":%d", TLSServerPort)) ||
+		strings.Contains(stderr, "raised from remote") {
+		return apiv1.BackupFailureReasonStandbyUnreachable
+	}
+	return apiv1.BackupFailureReasonObjectStoreError
+}
+
+// classifyArchiveTimeoutError separates a WAL segment that did not reach the
+// repository from a standby that did not replay to the backup start LSN in
+// time. Only the first one is a WAL archiving failure.
+func classifyArchiveTimeoutError(stderr string) apiv1.BackupFailureReason {
+	if strings.Contains(stderr, "was not archived before") {
+		return apiv1.BackupFailureReasonWalArchiving
+	}
+	return apiv1.BackupFailureReasonPgBackRestError
+}
+
+func isVersionMismatch(stderr string) bool {
+	return strings.Contains(stderr, "greeting key 'version'")
+}
+
 // CommandError is returned when a pgbackrest command fails.
 type CommandError struct {
 	// Command is the pgbackrest subcommand that failed (e.g. "archive-push").

pkg/pgbackrest/command_test.go

@@ -268,6 +268,20 @@ func TestClassifyExitCode(t *testing.T) {
 			expected: apiv1.BackupFailureReasonVersionMismatch,
 			ok:       true,
 		},
+		{
+			name: "version mismatch, exit 56, standby handshake fails during a mixed rollout",
+			err: &CommandError{
+				Command:  "backup",
+				ExitCode: 56,
+				Stderr: "WARN: unable to check pg2: [ProtocolError] expected value '2.59.1' for greeting key " +
+					"'version' but got '2.59.0'\n" +
+					"HINT: is the same version of pgBackRest installed on the local and remote host?\n" +
+					"ERROR: [056]: unable to find primary cluster - cannot proceed\n" +
+					"HINT: are all available clusters in recovery?\n",
+			},
+			expected: apiv1.BackupFailureReasonVersionMismatch,
+			ok:       true,
+		},
 		{
 			name: "standby unreachable, exit 39, TLS connect failure on port 8432",
 			err: &CommandError{
@@ -288,6 +302,55 @@ func TestClassifyExitCode(t *testing.T) {
 			expected: apiv1.BackupFailureReasonStandbyUnreachable,
 			ok:       true,
 		},
+		{
+			name: "object store error, exit 49, the S3 endpoint refuses the connection",
+			err: &CommandError{
+				Command:  "backup",
+				ExitCode: 49,
+				Stderr: "ERROR: [049]: unable to load info file '/c/backup/c/backup.info' or " +
+					"'/c/backup/c/backup.info.copy':\n" +
+					"HostConnectError: unable to connect to 'minio.example.svc:9000 (10.0.0.9)': " +
+					"[111] Connection refused\n" +
+					"[HostConnectError] on 10 retries from 105-60001ms: unable to connect to " +
+					"'minio.example.svc:9000 (10.0.0.9)': [111] Connection refused\n" +
+					"HINT: backup.info cannot be opened and is required to perform a backup.\n",
+			},
+			expected: apiv1.BackupFailureReasonObjectStoreError,
+			ok:       true,
+		},
+		{
+			name: "standby unreachable, exit 49, error raised from a remote pg host",
+			err: &CommandError{
+				Command:  "backup",
+				ExitCode: 49,
+				Stderr: "ERROR: [049]: raised from remote-0 tls protocol on 'cluster-example-2': " +
+					"unable to connect to 'cluster-example-2.svc (10.0.0.2)': [111] Connection refused",
+			},
+			expected: apiv1.BackupFailureReasonStandbyUnreachable,
+			ok:       true,
+		},
+		{
+			name: "wal archiving, exit 82, the backup stop segment never reached the repository",
+			err: &CommandError{
+				Command:  "backup",
+				ExitCode: 82,
+				Stderr: "ERROR: [082]: WAL segment 000000010000000000000005 was not archived before the 60000ms timeout\n" +
+					"HINT: check the archive_command to ensure that all options are correct (especially --stanza).",
+			},
+			expected: apiv1.BackupFailureReasonWalArchiving,
+			ok:       true,
+		},
+		{
+			name: "generic pgbackrest error, exit 82, standby does not replay in time",
+			err: &CommandError{
+				Command:  "backup",
+				ExitCode: 82,
+				Stderr: "ERROR: [082]: timeout before standby replayed to 0/5000028 - only reached 0/4000000\n" +
+					"HINT: is replication running and current on the standby?",
+			},
+			expected: apiv1.BackupFailureReasonPgBackRestError,
+			ok:       true,
+		},
 		{
 			name: "generic pgbackrest error for an exit code with no dedicated reason",
 			err: &CommandError{