Update root CA certificate ownership kuttl test

Adds better check logic to account for potential race conditions
that may be encountered in some environments due to delays in
garbage collection and ownership updating. Also fixed a comment
and harmonized filenames with existing patterns.

Author: tjmoore4

testing/kuttl/e2e/root-cert-ownership/01--check-owners.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Get a list of the current owners of the root ca cert secret and verify that
+  # both owners are listed.
+  - script: |
+      for i in {1..5}; do
+          sleep 1 # this sleep allows time for the owner reference list to be updated
+          CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
+            pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
+          # If owner1 and owner2 are both listed, exit successfully
+          if [[ "$CURRENT_OWNERS" == *"owner1"* ]] && [[ "$CURRENT_OWNERS" == *"owner2"* ]]; then
+              exit 0
+          fi
+      done
+      # proper ownership references were not found, so the test fails
+      exit 1

testing/kuttl/e2e/root-cert-ownership/01-check-owners.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Get a list of the current owners of the root ca cert secret and verify that
+  # owner1 is no longer listed and owner2 is found.
+  - script: |
+      for i in {1..5}; do
+          sleep 1 # this sleep allows time for the owner reference list to be updated
+          CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
+            pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
+          # If owner1 is removed and owner2 is still listed, exit successfully
+          if [[ "$CURRENT_OWNERS" != *"owner1"* ]] && [[ "$CURRENT_OWNERS" == *"owner2"* ]]; then
+              exit 0
+          fi
+      done
+      # proper ownership references were not found, so the test fails
+      exit 1

testing/kuttl/e2e/root-cert-ownership/03--check-owners.yaml

@@ -2,25 +2,33 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  # If there are other PostgresClusters in the namespace, ensure that 'owner2'
+  # If there are other PostgresClusters in the namespace, ensure that 'owner1'
   # and 'owner2' are not listed.
   # If there are no other PostgresClusters in the namespace, the 'pgo-root-cacert'
   # secret should be deleted.
   - script: |
       NUM_CLUSTERS=$(kubectl --namespace="${NAMESPACE}" get postgrescluster --output name | wc -l)
       if [[ "$NUM_CLUSTERS" != 0 ]]; then
-          CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
-            pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
-          if [[ "$CURRENT_OWNERS" == *"owner1"* ]]; then
-              exit 1
-          fi
-          if [[ "$CURRENT_OWNERS" == *"owner2"* ]]; then
-              exit 1
-          fi
+          for i in {1..5}; do
+              sleep 1 # This sleep allows time for the owner reference list to be updated
+              CURRENT_OWNERS=$(kubectl --namespace="${NAMESPACE}" get secret \
+                pgo-root-cacert -o jsonpath='{.metadata.ownerReferences[*].name}')
+              # If neither owner is listed, exit successfully
+              if [[ "$CURRENT_OWNERS" != *"owner1"* ]] || [[ "$CURRENT_OWNERS" != *"owner2"* ]]; then
+                  exit 0
+              fi
+          done
+          # At least one owner was never removed, so the test fails
+          exit 1
       else
-          ROOT_SECRET=$(kubectl --namespace="${NAMESPACE}" get --ignore-not-found \
-            secret pgo-root-cacert --output name | wc -l)
-          if [[ "$ROOT_SECRET" != 0 ]]; then
-              exit 1
-          fi
+          for i in {1..5}; do
+              sleep 1 # this sleep allows time for garbage collector to delete the secret
+              ROOT_SECRET=$(kubectl --namespace="${NAMESPACE}" get --ignore-not-found \
+                secret pgo-root-cacert --output name | wc -l)
+              if [[ "$ROOT_SECRET" == 0 ]]; then
+                  exit 0
+              fi
+          done
+          # The root secret was never removed, so the test fails
+          exit 1
       fi