#125: Added retry when deleting SaaS instance

.github/workflows/run-tests.yml

@@ -39,10 +39,8 @@ jobs:
           SAAS_ACCOUNT_ID: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_ACCOUNT_ID }}
           SAAS_PAT: ${{ secrets.INTEGRATION_TEAM_SAAS_STAGING_PAT }}
           PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}'
-        run: |
-          echo "PYTEST_ADDOPTS = $PYTEST_ADDOPTS"
-          export PROJECT_SHORT_TAG=$(poetry run -- nox -s project:get-short-tag)
-          poetry run -- nox -s test:coverage --
+          PROJECT_SHORT_TAG: SAPIPY
+        run: poetry run -- nox -s test:coverage --
 
       - name: Upload Artifacts
         uses: actions/upload-artifact@v4.6.2

doc/changes/changelog.md

@@ -1,6 +1,7 @@
 # Changes
 
 * [unreleased](unreleased.md)
+* [2.6.0](changes_2.6.0.md)
 * [2.5.0](changes_2.5.0.md)
 * [2.4.0](changes_2.4.0.md)
 * [2.3.0](changes_2.3.0.md)
@@ -27,6 +28,7 @@
 hidden:
 ---
 unreleased
+changes_2.6.0
 changes_2.5.0
 changes_2.4.0
 changes_2.3.0

doc/changes/changes_2.6.0.md

@@ -0,0 +1,11 @@
+# 2.6.0 - 2025-12-11
+
+## Summary
+
+This release changes the strategy for deleting a SaaS database instance, as required by the changed behavior of the actual SaaS backend.
+
+Before, SAPIPY used only a fixed waiting time. In case of HTTP responses with status code 400 and message `Operation.*not allowed.*cluster.*not.*in.*proper state` SAPIPY now retries deleting the SaaS instance for max. 5 minutes.
+
+## Bugfixes
+
+* #125: Added retry when deleting SaaS instance

exasol/saas/client/api_access.py

@@ -3,21 +3,28 @@
 import datetime as dt
 import getpass
 import logging
+import re
 import time
 from collections.abc import Iterable
 from contextlib import contextmanager
 from datetime import (
     datetime,
     timedelta,
+    timezone,
 )
 from typing import Any
 
+import tenacity
 from tenacity import (
     TryAgain,
     retry,
 )
+from tenacity.retry import retry_if_exception
 from tenacity.stop import stop_after_delay
-from tenacity.wait import wait_fixed
+from tenacity.wait import (
+    wait_exponential,
+    wait_fixed,
+)
 
 from exasol.saas.client import (
     Limits,
@@ -38,31 +45,39 @@
     delete_allowed_ip,
     list_allowed_i_ps,
 )
+from exasol.saas.client.openapi.errors import UnexpectedStatus
+from exasol.saas.client.openapi.models.exasol_database import ExasolDatabase
 from exasol.saas.client.openapi.models.status import Status
 from exasol.saas.client.openapi.types import UNSET
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.INFO)
 
 
+def interval_retry(interval: timedelta, timeout: timedelta):
+    return tenacity.retry(wait=wait_fixed(interval), stop=stop_after_delay(timeout))
+
+
 def timestamp_name(project_short_tag: str | None = None) -> str:
     """
     project_short_tag: Abbreviation of your project
     """
-    timestamp = f"{datetime.now().timestamp():.0f}"
+    timestamp = f"{datetime.now(timezone.utc).timestamp():.0f}"
     owner = getpass.getuser()
     candidate = f"{timestamp}{project_short_tag or ''}-{owner}"
     return candidate[: Limits.MAX_DATABASE_NAME_LENGTH]
 
 
-def wait_for_delete_clearance(start: dt.datetime):
-    lifetime = datetime.now() - start
-    if lifetime < Limits.MIN_DATABASE_LIFETIME:
-        wait = Limits.MIN_DATABASE_LIFETIME - lifetime
-        LOG.info(
-            f"Waiting {int(wait.seconds)} seconds" " before deleting the database."
-        )
-        time.sleep(wait.seconds)
+def indicates_retry(ex: BaseException) -> bool:
+    """
+    When deleting a SaaS instance raises an UnexpectedStatus, then this
+    function decides whether we should retry to delete the database instance.
+    """
+    return bool(
+        isinstance(ex, UnexpectedStatus)
+        and ex.status_code == 400
+        and "cluster is not in a proper state" in ex.content.decode("utf-8")
+    )
 
 
 class DatabaseStartupFailure(Exception):
@@ -202,7 +217,7 @@ def create_database(
         cluster_size: str = "XS",
         region: str = "eu-central-1",
         idle_time: timedelta | None = None,
-    ) -> openapi.models.exasol_database.ExasolDatabase | None:
+    ) -> ExasolDatabase | None:
         def minutes(x: timedelta) -> int:
             return x.seconds // 60
 
@@ -215,7 +230,7 @@ def minutes(x: timedelta) -> int:
                 idle_time=minutes(idle_time),
             ),
         )
-        LOG.info(f"Creating database {name}")
+        LOG.info("Creating database %s", name)
         return create_database.sync(
             self._account_id,
             client=self._client,
@@ -241,7 +256,7 @@ def wait_until_deleted(
         timeout: timedelta = timedelta(seconds=1),
         interval: timedelta = timedelta(minutes=1),
     ):
-        @retry(wait=wait_fixed(interval), stop=stop_after_delay(timeout))
+        @interval_retry(interval, timeout)
         def still_exists() -> bool:
             result = database_id in self.list_database_ids()
             if result:
@@ -251,12 +266,45 @@ def still_exists() -> bool:
         if still_exists():
             raise DatabaseDeleteTimeout
 
-    def delete_database(self, database_id: str, ignore_failures=False):
-        with self._ignore_failures(ignore_failures) as client:
-            return delete_database.sync_detailed(
-                self._account_id, database_id, client=client
+    def delete_database(
+        self,
+        database_id: str,
+        ignore_failures: bool = False,
+        timeout: timedelta = timedelta(minutes=30),
+        min_interval: timedelta = timedelta(seconds=1),
+        max_interval: timedelta = timedelta(minutes=2),
+    ) -> None:
+        @retry(
+            reraise=True,
+            wait=wait_exponential(
+                multiplier=1,
+                min=min_interval,
+                max=max_interval,
+            ),
+            stop=stop_after_delay(timeout),
+            retry=retry_if_exception(indicates_retry),
+        )
+        def delete_with_retry():
+            LOG.info("Trying to delete database with ID %s ...", database_id)
+            delete_database.sync_detailed(
+                self._account_id,
+                database_id,
+                client=self._client,
             )
 
+        try:
+            delete_with_retry()
+            LOG.info("Deleted database with ID %s", database_id)
+        except Exception as ex:
+            if ignore_failures:
+                LOG.error(
+                    "Ignoring failure when deleting database with ID %s: %s",
+                    database_id,
+                    ex,
+                )
+            else:
+                raise
+
     def list_database_ids(self) -> Iterable[str]:
         dbs = list_databases.sync(self._account_id, client=self._client) or []
         return (db.id for db in dbs)
@@ -270,29 +318,21 @@ def database(
         idle_time: timedelta | None = None,
     ):
         db = None
-        start = datetime.now()
         try:
             db = self.create_database(name, idle_time=idle_time)
             yield db
-            wait_for_delete_clearance(start)
         finally:
             db_repr = f"{db.name} with ID {db.id}" if db else None
-            if db and not keep:
-                LOG.info(f"Deleting database {db_repr}")
-                response = self.delete_database(db.id, ignore_delete_failure)
-                if response.status_code == 200:
-                    LOG.info(f"Successfully deleted database {db_repr}.")
-                else:
-                    LOG.warning(f"Ignoring status code {response.status_code}.")
-            elif not db:
-                LOG.warning("Cannot delete db None")
+            if not db:
+                LOG.warning("Cannot delete database None")
+            elif keep:
+                LOG.info("Keeping database %s as keep = %s.", db_repr, keep)
             else:
-                LOG.info(f"Keeping database {db_repr} as keep = {keep}")
+                LOG.info("Deleting database %s", db_repr)
+                self.delete_database(db.id, ignore_delete_failure)
+                LOG.info("Successfully deleted database %s.", db_repr)
 
-    def get_database(
-        self,
-        database_id: str,
-    ) -> openapi.models.exasol_database.ExasolDatabase | None:
+    def get_database(self, database_id: str) -> ExasolDatabase | None:
         return get_database.sync(
             self._account_id,
             database_id,
@@ -305,17 +345,16 @@ def wait_until_running(
         timeout: timedelta = timedelta(minutes=30),
         interval: timedelta = timedelta(minutes=2),
     ):
-        success = [
-            Status.RUNNING,
-        ]
+        success = [Status.RUNNING]
 
-        @retry(wait=wait_fixed(interval), stop=stop_after_delay(timeout))
-        def poll_status():
+        @interval_retry(interval, timeout)
+        def poll_status() -> Status:
             db = self.get_database(database_id)
-            if db.status not in success:
-                LOG.info("- Database status: %s ...", db.status)
+            status = db.status if db else None
+            if status not in success:
+                LOG.info("- Database status: %s ...", status)
                 raise TryAgain
-            return db.status
+            return status
 
         if poll_status() not in success:
             raise DatabaseStartupFailure()

noxfile.py

@@ -127,18 +127,3 @@ def check_api_outdated(session: Session):
     """
     generate_api(session)
     session.run("git", "diff", "--exit-code", DEST_DIR)
-
-
-@nox.session(name="project:get-short-tag", python=False)
-def get_project_short_tag(session: Session):
-    config_file = Path("error_code_config.yml")
-    content = config_file.read_text()
-    header = False
-    for line in content.splitlines():
-        line = line.strip()
-        if header:
-            print(line.strip().replace(":", ""))
-            return
-        if line.startswith("error-tags:"):
-            header = True
-    raise RuntimeError(f"Could not read project short tag from file {config_file}")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "exasol-saas-api"
-version = "2.5.0"
+version = "2.6.0"
 requires-python = ">=3.10.0,<4.0"
 description = "API enabling Python applications connecting to Exasol database SaaS instances and using their SaaS services"
 authors = [

test/integration/allowed_ip_test.py

@@ -1,3 +1,7 @@
+import pytest
+
+
+@pytest.mark.slow
 def test_lifecycle(api_access):
     testee = api_access
     with testee.allowed_ip(ignore_delete_failure=True) as ip:

test/integration/conftest.py

@@ -59,12 +59,12 @@ def operational_saas_database_id(api_access, database_name) -> str:
 
 
 @pytest.fixture(scope="session")
-def project_short_tag():
-    return os.environ.get("PROJECT_SHORT_TAG")
+def project_short_tag() -> str:
+    return "SAPIPY"
 
 
 @pytest.fixture(scope="session")
-def database_name(project_short_tag):
+def database_name(project_short_tag) -> str:
     return timestamp_name(project_short_tag)
 
 

test/integration/connection_test.py

@@ -1,47 +1,57 @@
+import contextlib
+import ssl
+
 import pyexasol
 import pytest
 
 from exasol.saas.client.api_access import get_connection_params
 
+SSL_OPTIONS = {"websocket_sslopt": {"cert_reqs": ssl.CERT_NONE}}
+
+
+@pytest.fixture
+def pyexasol_connection(
+    saas_host,
+    saas_pat,
+    saas_account_id,
+    allow_connection,
+    operational_saas_database_id,
+):
+    @contextlib.contextmanager
+    def connect(**kwargs):
+        params = get_connection_params(
+            host=saas_host,
+            account_id=saas_account_id,
+            pat=saas_pat,
+            **kwargs,
+        )
+        params.update(SSL_OPTIONS)
+        with pyexasol.connect(**params) as con:
+            yield con
+
+    return connect
+
 
 @pytest.mark.slow
 def test_get_connection_params_with_id(
-    saas_host, saas_pat, saas_account_id, operational_saas_database_id, allow_connection
+    pyexasol_connection,
+    operational_saas_database_id,
 ):
     """
     This integration test checks that opening a pyexasol connection to a SaaS DB with
     known id and executing a query works.
     """
-    connection_params = get_connection_params(
-        host=saas_host,
-        account_id=saas_account_id,
-        pat=saas_pat,
-        database_id=operational_saas_database_id,
-    )
-    with pyexasol.connect(**connection_params) as pyconn:
+    with pyexasol_connection(database_id=operational_saas_database_id) as pyconn:
         result = pyconn.execute("SELECT 1;").fetchall()
         assert result == [(1,)]
 
 
 @pytest.mark.slow
-def test_get_connection_params_with_name(
-    saas_host,
-    saas_pat,
-    saas_account_id,
-    operational_saas_database_id,
-    database_name,
-    allow_connection,
-):
+def test_get_connection_params_with_name(pyexasol_connection, database_name):
     """
     This integration test checks that opening a pyexasol connection to a SaaS DB with
     known name and executing a query works.
     """
-    connection_params = get_connection_params(
-        host=saas_host,
-        account_id=saas_account_id,
-        pat=saas_pat,
-        database_name=database_name,
-    )
-    with pyexasol.connect(**connection_params) as pyconn:
+    with pyexasol_connection(database_name=database_name) as pyconn:
         result = pyconn.execute("SELECT 1;").fetchall()
         assert result == [(1,)]