Extended e2e for all shared, workspaces, and workspace services

Fixes microsoft#1921
marrobi · Oct 17, 2023 · 5bbad91 · 5bbad91
1 parent a7436dd
commit 5bbad91
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 12 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -135,6 +135,22 @@
                 "false"
               ]
             },
+            {
+              "name": "E2E Workspace Services",
+              "type": "python",
+              "request": "launch",
+              "module": "pytest",
+              "justMyCode": true,
+              "cwd": "${workspaceFolder}/e2e_tests/",
+              "preLaunchTask": "Copy_env_file_for_e2e_debug",
+              "args": [
+                "-n 5",
+                "-m",
+                "workspace_services",
+                "--verify",
+                "false"
+              ]
+            },
             {
               "name": "E2E Performance",
               "type": "python",

diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml
@@ -32,7 +32,7 @@ jobs:
         ${{ (github.event_name == 'push' && 'extended or extended_aad')
         || 'extended or extended_aad or shared_services or airlock' }}
       environmentName: ${{ github.event.inputs.environment || 'CICD' }}
-      E2E_TESTS_NUMBER_PROCESSES: 1
+      E2E_TESTS_NUMBER_PROCESSES: 5
       DEVCONTAINER_TAG: 'latest'
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}

diff --git a/.github/workflows/pr_comment_bot.yml b/.github/workflows/pr_comment_bot.yml
@@ -152,7 +152,7 @@ jobs:
         (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') ||
         (needs.pr_comment.outputs.command == 'run-tests' && '') }}
       environmentName: CICD
-      E2E_TESTS_NUMBER_PROCESSES: 1
+      E2E_TESTS_NUMBER_PROCESSES: 5
       DEVCONTAINER_TAG: ${{ needs.pr_comment.outputs.prRefId }}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}

diff --git a/api_app/_version.py b/api_app/_version.py
@@ -1 +1 @@
-__version__ = "0.15.17"
+__version__ = "0.15.18"
diff --git a/api_app/api/routes/workspaces.py b/api_app/api/routes/workspaces.py
@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import time
 
 from fastapi import APIRouter, Depends, HTTPException, Header, status, Request, Response
 
@@ -256,12 +257,25 @@ async def create_workspace_service(response: Response, workspace_service_input:
         workspace_patch = ResourcePatch()
         workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]}
         # IP address allocation is managed by the API. Ideally this request would happen as a result of the workspace
-        # service deployment via the reosurce processor. there is no such functionality so the database is being
+        # service deployment via the resource processor. there is no such functionality so the database is being
         # updated directly, and an "update" on the workspace is called by the workspace service pipeline.
-        try:
-            await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False)
-        except CosmosAccessConditionFailedError:
-            raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT)
+        MAX_RETRIES = 3
+        RETRY_DELAY = 1
+
+        retries = 0
+        while retries < MAX_RETRIES:
+            try:
+                await workspace_repo.patch_workspace(workspace, workspace_patch, workspace.etag, resource_template_repo, resource_history_repo, user, False)
+                break
+            except CosmosAccessConditionFailedError:
+                workspace = await workspace_repo.get_workspace(workspace.id)
+                workspace.etag = workspace.etag.strip('"')
+                workspace_patch = ResourcePatch()
+                workspace_patch.properties = {"address_spaces": workspace.properties["address_spaces"] + [workspace_service.properties["address_space"]]}
+                retries += 1
+                if retries == MAX_RETRIES:
+                    raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=strings.ETAG_CONFLICT)
+                time.sleep(RETRY_DELAY)
 
     operation = await save_and_deploy_resource(
         resource=workspace_service,

diff --git a/e2e_tests/resources/strings.py b/e2e_tests/resources/strings.py
@@ -14,12 +14,15 @@
 AIRLOCK_IMPORT_REVIEW_WORKSPACE = "tre-workspace-airlock-import-review"
 
 AZUREML_SERVICE = "tre-service-azureml"
-INNEREYE_SERVICE = "tre-service-innereye"
-GUACAMOLE_SERVICE = "tre-service-guacamole"
+DATABRICKS_SERVICE = "tre-service-databricks"
 GITEA_SERVICE = "tre-workspace-service-gitea"
+GUACAMOLE_SERVICE = "tre-service-guacamole"
+INNEREYE_SERVICE = "tre-service-innereye"
+HEALTH_SERVICE = "tre-workspace-service-health"
 MLFLOW_SERVICE = "tre-service-mlflow"
 MYSQL_SERVICE = "tre-workspace-service-mysql"
-HEALTH_SERVICE = "tre-workspace-service-health"
+OHDSI_SERVICE = "tre-workspace-service-ohdsi"
+
 
 FIREWALL_SHARED_SERVICE = "tre-shared-service-firewall"
 GITEA_SHARED_SERVICE = "tre-shared-service-gitea"

diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py
@@ -9,8 +9,11 @@
 
 workspace_services = [
     strings.AZUREML_SERVICE,
-    # strings.INNEREYE_SERVICE,
+    strings.DATABRICKS_SERVICE,
+    strings.GUACAMOLE_SERVICE,
     strings.GITEA_SERVICE,
+    # strings.INNEREYE_SERVICE,
+    # strings.OHDSI_SERVICE,
     strings.MLFLOW_SERVICE,
     strings.MYSQL_SERVICE,
     strings.HEALTH_SERVICE,
@@ -63,6 +66,7 @@ async def ping_guacamole_workspace_service(workspace_service_path, access_token,
     await check_aad_auth_redirect(endpoint, verify)
 
 
+@pytest.mark.extended
 @pytest.mark.workspace_services
 @pytest.mark.timeout(45 * 60)
 @pytest.mark.parametrize("template_name", workspace_services)