test_mcg_namespace_object_versions_crd[bucketclass_dict0] - test case fails in 50% of test runs with consistent error

[ypersky1980]

test_mcg_namespace_object_versions_crd[bucketclass_dict0] - test case fails in 50% of test runs with consistent error , an investigation and a fix is needed.

Relevant runs:
https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/795/27647/1363270/1363351/log
https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/795/28022/1376497/1376578/log
https://reportportal-ocs4.apps.ocp-c1.prod.psi.redhat.com/ui/#ocs/launches/795/28535/1401296/1401377/log

Backtrace:

self = <ocs_ci.ocs.resources.namespacestore.NamespaceStore object at 0x7f116f2445b0>
timeout = 180, interval = 5

def verify_health(self, timeout=180, interval=5):
"""
Health verification function that tries to verify
a namespacestores's health until a given time limit is reached

Args:
    timeout (int): Timeout for the check, in seconds
    interval (int): Interval to wait between checks, in seconds

Returns:
    (bool): True if the bucket is healthy, False otherwise

"""
log.info(f"Waiting for {self.name} to be healthy")
try:

  for health_check in TimeoutSampler(
        timeout, interval, getattr(self, f"{self.method}_verify_health")
    ):

/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/ocs_ci/ocs/resources/namespacestore.py:174:

self = <ocs_ci.utility.utils.TimeoutSampler object at 0x7f116bcedcd0>

def iter(self):
if self.start_time is None:
self.start_time = time.time()
while True:
self.last_sample_time = time.time()
if self.timeout <= (self.last_sample_time - self.start_time):

      raise self.timeout_exc_cls(*self.timeout_exc_args)

E ocs_ci.ocs.exceptions.TimeoutExpiredError: Timed out after 180s running oc_verify_health()

/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/ocs_ci/utility/utils.py:1481: TimeoutExpiredError

During handling of the above exception, another exception occurred:

self = <test_mcg_namespace_s3_ops_crd.TestMcgNamespaceS3OperationsCrd object at 0x7f11968415e0>
mcg_obj = <ocs_ci.ocs.resources.mcg.MCG object at 0x7f116eb9ea30>
cld_mgr = <ocs_ci.ocs.resources.cloud_manager.CloudManager object at 0x7f11bd7f71c0>
bucket_factory = <function bucket_factory_fixture.._create_buckets at 0x7f1189cc6790>
bucketclass_dict = {'interface': 'OC', 'namespace_policy_dict': {'namespacestore_dict': {'aws': [(1, 'eu-central-1')]}, 'type': 'Single'}}

@pytest.mark.parametrize(
argnames=["bucketclass_dict"],
argvalues=[
pytest.param(
{
"interface": "OC",
"namespace_policy_dict": {
"type": "Single",
"namespacestore_dict": {"aws": [(1, "eu-central-1")]},
},
}
),
],
)
def test_mcg_namespace_object_versions_crd(
self, mcg_obj, cld_mgr, bucket_factory, bucketclass_dict
):
"""
Test object versioning S3 operations on namespace buckets/resources(CRDs).
Validates put, get, delete object version operations

"""
obj_versions = []
version_key = "ObjKey-" + str(uuid.uuid4().hex)
total_versions = 10
aws_s3_resource = boto3.resource(
    "s3",
    endpoint_url=constants.MCG_NS_AWS_ENDPOINT,
    aws_access_key_id=cld_mgr.aws_client.access_key,
    aws_secret_access_key=cld_mgr.aws_client.secret_key,
)

ns_buc = bucket_factory(
amount=1,
interface=bucketclass_dict["interface"],
bucketclass=bucketclass_dict,
)[0]

/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/tests/functional/object/mcg/lifecycle/test_mcg_namespace_s3_ops_crd.py:471:

/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/tests/conftest.py:2992: in _create_buckets
bucketclass if bucketclass is None else bucket_class_factory(bucketclass)
/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/ocs_ci/ocs/resources/bucketclass.py:134: in _create_bucket_class
namespacestores = namespace_store_factory(interface, nss_dict)
/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/ocs_ci/ocs/resources/namespacestore.py:466: in _create_nss
nss_obj.verify_health()

self = <ocs_ci.ocs.resources.namespacestore.NamespaceStore object at 0x7f116f2445b0>
timeout = 180, interval = 5

def verify_health(self, timeout=180, interval=5):
"""
Health verification function that tries to verify
a namespacestores's health until a given time limit is reached

Args:
    timeout (int): Timeout for the check, in seconds
    interval (int): Interval to wait between checks, in seconds

Returns:
    (bool): True if the bucket is healthy, False otherwise

"""
log.info(f"Waiting for {self.name} to be healthy")
try:
    for health_check in TimeoutSampler(
        timeout, interval, getattr(self, f"{self.method}_verify_health")
    ):
        if health_check:
            log.info(f"{self.name} is healthy")
            return True
        else:
            log.info(f"{self.name} is unhealthy. Rechecking.")
except TimeoutExpiredError:
    log.error(
        f"{self.name} did not reach a healthy state within {timeout} seconds."
    )

  assert (
        False
    ), f"{self.name} did not reach a healthy state within {timeout} seconds."

E AssertionError: aws-ns-store-06e5475757ff4a849ecf61754ff did not reach a healthy state within 180 seconds.

/home/jenkins/workspace/qe-deploy-ocs-cluster-prod/ocs-ci/ocs_ci/ocs/resources/namespacestore.py:186: AssertionError