EVEREST-1403 | Fix broken uninstallation after 1.2.0 API migration (#632

) As a part of the 1.2.0 upgrade, we perform a migration of BackupStorages and MonitoringConfigs. Prior to 1.2.0, the BackupStorages have a `percona.com/cleanup-secrets` finalizer that was intended to remove Secrets in the DB namespaces. Uninstallation gets stuck because this finalizer is no longer handled by the operator **Solution:** * Remove the finalizer during the migration, since the Secrets in the DB namespaces are now owned by the newly created local BackupStorage * Also fix the uninstall code to list BackupStorages/MonitoringConfigs from DB namespaces. Signed-off-by: Mayank Shah <[email protected]>
percona · Sep 3, 2024 · 4b2d716 · 4b2d716
1 parent a5bc3f0
commit 4b2d716
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 20 deletions.
diff --git a/pkg/uninstall/uninstall.go b/pkg/uninstall/uninstall.go
@@ -380,6 +380,19 @@ func (u *Uninstall) deleteBackupStorages(ctx context.Context) error {
 		return err
 	}
 
+	// List backup storages in DB namespaces.
+	dbNamespaces, err := u.kubeClient.GetDBNamespaces(ctx)
+	if err != nil {
+		return err
+	}
+	for _, ns := range dbNamespaces {
+		list, err := u.kubeClient.ListBackupStorages(ctx, ns)
+		if err != nil {
+			return fmt.Errorf("failed to list backup storages in namespace %s: %w", ns, err)
+		}
+		storages.Items = append(storages.Items, list.Items...)
+	}
+
 	if len(storages.Items) == 0 {
 		u.l.Info("All backup storages have been deleted")
 		return nil
@@ -388,25 +401,24 @@ func (u *Uninstall) deleteBackupStorages(ctx context.Context) error {
 	for _, storage := range storages.Items {
 		u.l.Infof("Deleting backup storage '%s'", storage.Name)
 		u.numResourcesDeleted++
-		if err := u.kubeClient.DeleteBackupStorage(ctx, common.SystemNamespace, storage.Name); err != nil {
+		if err := u.kubeClient.DeleteBackupStorage(ctx, storage.GetNamespace(), storage.GetName()); err != nil {
 			return err
 		}
 	}
 
 	// Wait for all backup storages to be deleted, or timeout after 5 minutes.
 	u.l.Infof("Waiting for backup storages to be deleted")
 	return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
-		storages, err := u.kubeClient.ListBackupStorages(ctx, common.SystemNamespace)
-		if err != nil {
-			return false, err
-		}
-
-		if len(storages.Items) > 0 {
+		for _, bs := range storages.Items {
+			_, err := u.kubeClient.GetBackupStorage(ctx, bs.GetNamespace(), bs.GetName())
+			if k8serrors.IsNotFound(err) {
+				continue
+			} else if err != nil {
+				return false, err
+			}
 			return false, nil
 		}
-
 		u.l.Info("All backup storages have been deleted")
-
 		return true, nil
 	})
 }
@@ -419,6 +431,18 @@ func (u *Uninstall) deleteMonitoringConfigs(ctx context.Context) error {
 		return err
 	}
 
+	dbNamespaces, err := u.kubeClient.GetDBNamespaces(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get DB namespaces: %w", err)
+	}
+	for _, ns := range dbNamespaces {
+		list, err := u.kubeClient.ListMonitoringConfigs(ctx, ns)
+		if err != nil {
+			return fmt.Errorf("failed to list monitoring configs in namespace %s: %w", ns, err)
+		}
+		monitoringConfigs.Items = append(monitoringConfigs.Items, list.Items...)
+	}
+
 	if len(monitoringConfigs.Items) == 0 {
 		u.l.Info("No monitoring configs found")
 		return nil
@@ -427,25 +451,24 @@ func (u *Uninstall) deleteMonitoringConfigs(ctx context.Context) error {
 	for _, config := range monitoringConfigs.Items {
 		u.numResourcesDeleted++
 		u.l.Infof("Deleting monitoring config '%s'", config.Name)
-		if err := u.kubeClient.DeleteMonitoringConfig(ctx, install.MonitoringNamespace, config.Name); err != nil {
+		if err := u.kubeClient.DeleteMonitoringConfig(ctx, config.GetNamespace(), config.GetName()); err != nil {
 			return err
 		}
 	}
 
 	// Wait for all monitoring configs to be deleted, or timeout after 5 minutes.
 	u.l.Infof("Waiting for monitoring configs to be deleted")
 	return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
-		monitoringConfigs, err := u.kubeClient.ListMonitoringConfigs(ctx, install.MonitoringNamespace)
-		if err != nil {
-			return false, err
-		}
-
-		if len(monitoringConfigs.Items) > 0 {
+		for _, mc := range monitoringConfigs.Items {
+			_, err := u.kubeClient.GetMonitoringConfig(ctx, mc.GetNamespace(), mc.GetName())
+			if k8serrors.IsNotFound(err) {
+				continue
+			} else if err != nil {
+				return false, err
+			}
 			return false, nil
 		}
-
 		u.l.Info("All monitoring configs have been deleted")
-
 		return true, nil
 	})
 }

diff --git a/pkg/upgrade/migrate_shared_resources.go b/pkg/upgrade/migrate_shared_resources.go
@@ -30,6 +30,10 @@ import (
 	"github.com/percona/everest/pkg/common"
 )
 
+const (
+	backupStorageCleanupFinalizer = "percona.com/cleanup-secrets"
+)
+
 //nolint:mnd
 func (u *Upgrade) migrateSharedResources(ctx context.Context) error {
 	var b backoff.BackOff
@@ -68,7 +72,6 @@ func (u *Upgrade) copySecret(ctx context.Context, secret *corev1.Secret, namespa
 	return nil
 }
 
-//nolint:dupl
 func (u *Upgrade) migrateBackupStorages(ctx context.Context) error {
 	backupStorages, err := u.kubeClient.ListBackupStorages(ctx, common.SystemNamespace)
 	if err != nil {
@@ -102,12 +105,27 @@ func (u *Upgrade) migrateBackupStorages(ctx context.Context) error {
 			if err := u.kubeClient.CreateBackupStorage(ctx, bsClone); client.IgnoreAlreadyExists(err) != nil {
 				return fmt.Errorf("cannot create backup storage %s in namespace %s", bsClone.GetName(), ns)
 			}
+
+			// Remove the Secret clean-up finalizer from the original BackupStorage.
+			// The purpose of this finalizer is to remove all Secrets linked to this BS, in the DB namespaces.
+			// After this migration, these Secrets are owned by the newly created BackupStorage in the target namespace.
+			finalizers := make([]string, 0, len(bs.GetFinalizers()))
+			for _, f := range bs.GetFinalizers() {
+				if f == backupStorageCleanupFinalizer {
+					continue
+				}
+				finalizers = append(finalizers, f)
+			}
+			bs.SetFinalizers(finalizers)
+			if err := u.kubeClient.UpdateBackupStorage(ctx, &bs); err != nil {
+				return fmt.Errorf("cannot remove finalizer %s from backup storage %s", backupStorageCleanupFinalizer, bs.Name)
+			}
+			return nil
 		}
 	}
 	return nil
 }
 
-//nolint:dupl
 func (u *Upgrade) migrateMonitoringInstaces(ctx context.Context) error {
 	monitoringConfigs, err := u.kubeClient.ListMonitoringConfigs(ctx, common.MonitoringNamespace)
 	if err != nil {