[Fabric E2E Sample] Adding CI pipelines (#1013)

* Version 2.0 changes
* CI changes
* Notebook changes
* AzDo pipelines automation
* Terraform upgrade
---------

Co-authored-by: Anuj Parashar <[email protected]>
Co-authored-by: Lace Lofranco <[email protected]>
Co-authored-by: yuna-s <[email protected]>
Co-authored-by: Naga Nandyala <[email protected]>

e2e_samples/fabric_dataops_sample/.envtemplate

@@ -1,25 +1,40 @@
-export TENANT_ID="The Entra ID (Azure AD Tenant Id) of your Fabric tenant"
-export SUBSCRIPTION_ID="The Azure subscription ID that will be used to deploy azure resources."
+# Environment variables for the Fabric DataOps sample. Some of the variable have sample values which can be used as-it. For others, please provide the actual values.
+## The Entra ID (Azure AD Tenant Id) of your Fabric tenant.
+export TENANT_ID=""
+## The Azure subscription ID that will be used to deploy azure resources.
+export SUBSCRIPTION_ID=""
 ## Space-separated array of deployment stages, development (dev), staging (stg), production (prod) etc., for your application.
+## Highly recommended to use the default values mentioned below.
 export ENVIRONMENT_NAMES=("dev" "stg" "prod")
 ## Space-separated array of 'pre-existing' Azure resource groups corresponding to each environment.
 export RESOURCE_GROUP_NAMES=("rg-dev" "rg-stg" "rg-prod")
-export BASE_NAME="The base name of the Fabric project. This name is used for naming the Azure and Fabric resources."
+## The base name of the Fabric project. This name is used for naming the Azure and Fabric resources.
+export BASE_NAME=""
 # Service principal authentication: For managed identity or Entra ID user authentication, you can leave the following two variables blank.
-export APP_CLIENT_ID="The service principal client ID."
-export APP_CLIENT_SECRET="The service principal client secret."
+## The service principal client ID.
+export APP_CLIENT_ID=""
+## The service principal client secret.
+export APP_CLIENT_SECRET=""
 # Azure DevOps variables (The project, repository, branch, and folder must be created in advance)
-export GIT_ORGANIZATION_NAME="The Azure DevOps organization."
-export GIT_PROJECT_NAME="The Azure Devops project."
-export GIT_REPOSITORY_NAME="Your repository under the Azure DevOps project."
+## The Azure DevOps organization.
+export GIT_ORGANIZATION_NAME=""
+## The Azure Devops project.
+export GIT_PROJECT_NAME=""
+## Your repository under the Azure DevOps project.
+export GIT_REPOSITORY_NAME=""
 ## Space-separated array of the GIT branches corresponding to each environment where Fabric items will be committed to.
+## Highly recommended to use the default values mentioned below.
 export GIT_BRANCH_NAMES=("dev" "stg" "prod")
-## Note: Other than the root folder "/", the directory must already exist. Must start with a forward-slash. Example: "/fabric"
-export GIT_DIRECTORY_NAME="The folder where Fabric items will be committed"
+## The username for the Azure DevOps project.
+export GIT_USERNAME=""
+## The personal access token of the above user.
+export GIT_PERSONAL_ACCESS_TOKEN=""
 # Workspace admin variables
-export FABRIC_WORKSPACE_ADMIN_SG_NAME="The name of the Entra security groups with admin members."
+## The name of the Entra security groups with admin members.
+export FABRIC_WORKSPACE_ADMIN_SG_NAME=""
 # Fabric Capacity variables
 ## The name of an existing Fabric capacity. If this is empty, then a new capacity will be created.
 export EXISTING_FABRIC_CAPACITY_NAME=""
 ## Comma-separated list. When creating a new Fabric capacity, these users/apps would be added as capacity admin. For users, mention "userPrincipalName". For principals (sp/mi), mention "Object ID". Don't add spaces after the comma.
-export FABRIC_CAPACITY_ADMINS="yourusername@yourdomain,sp_mi_object_id"
+## For example: "[email protected],sp_mi_object_id"
+export FABRIC_CAPACITY_ADMINS=""

e2e_samples/fabric_dataops_sample/cleanup.sh

@@ -12,9 +12,17 @@ exec > >(tee -a "$log_file")
 exec 2>&1
 
 for i in "${!ENVIRONMENT_NAMES[@]}"; do
+
+  if [ "$i" -eq 0 ]; then
+    deploy_fabric_items="true"
+  else
+    deploy_fabric_items="false"
+  fi
+
   ENVIRONMENT_NAME=${ENVIRONMENT_NAMES[$i]} \
   RESOURCE_GROUP_NAME=${RESOURCE_GROUP_NAMES[$i]} \
   TENANT_ID=$TENANT_ID \
+  SUBSCRIPTION_ID=$SUBSCRIPTION_ID \
   BASE_NAME=$BASE_NAME \
   APP_CLIENT_ID=$APP_CLIENT_ID \
   APP_CLIENT_SECRET=$APP_CLIENT_SECRET \
@@ -25,5 +33,6 @@ for i in "${!ENVIRONMENT_NAMES[@]}"; do
   FABRIC_WORKSPACE_ADMIN_SG_NAME=$FABRIC_WORKSPACE_ADMIN_SG_NAME \
   EXISTING_FABRIC_CAPACITY_NAME=$EXISTING_FABRIC_CAPACITY_NAME \
   FABRIC_CAPACITY_ADMINS=$FABRIC_CAPACITY_ADMINS \
+  DEPLOY_FABRIC_ITEMS=$deploy_fabric_items \
   bash -c "./scripts/cleanup_infrastructure.sh"
 done

e2e_samples/fabric_dataops_sample/deploy.sh

@@ -13,10 +13,26 @@ log_file="deploy_${BASE_NAME}_$(date +"%Y%m%d_%H%M%S").log"
 exec > >(tee -a "$log_file")
 exec 2>&1
 
+# Global variable to capture the first environment ("dev") branch name
+azdo_policies_branch_name=""
+# Global variable to capture the last environment ("prod") branch name
+azdo_pipelines_branch_name=""
+
 for i in "${!ENVIRONMENT_NAMES[@]}"; do
+
+  if [ "$i" -eq 0 ]; then
+    deploy_fabric_items="true"
+    azdo_policies_branch_name=${GIT_BRANCH_NAMES[$i]}
+  else
+    deploy_fabric_items="false"
+  fi
+
+  azdo_pipelines_branch_name=${GIT_BRANCH_NAMES[$i]}
+
   ENVIRONMENT_NAME=${ENVIRONMENT_NAMES[$i]} \
   RESOURCE_GROUP_NAME=${RESOURCE_GROUP_NAMES[$i]} \
   TENANT_ID=$TENANT_ID \
+  SUBSCRIPTION_ID=$SUBSCRIPTION_ID \
   BASE_NAME=$BASE_NAME \
   APP_CLIENT_ID=$APP_CLIENT_ID \
   APP_CLIENT_SECRET=$APP_CLIENT_SECRET \
@@ -27,5 +43,19 @@ for i in "${!ENVIRONMENT_NAMES[@]}"; do
   FABRIC_WORKSPACE_ADMIN_SG_NAME=$FABRIC_WORKSPACE_ADMIN_SG_NAME \
   EXISTING_FABRIC_CAPACITY_NAME=$EXISTING_FABRIC_CAPACITY_NAME \
   FABRIC_CAPACITY_ADMINS=$FABRIC_CAPACITY_ADMINS \
+  DEPLOY_FABRIC_ITEMS=$deploy_fabric_items \
   bash -c "./scripts/deploy_infrastructure.sh"
 done
+
+# Deploy Azure DevOps pipelines and create branch policies
+user_principal_type=$(az account show --query user.type -o tsv)
+if [ "$user_principal_type" == "user" ]; then
+  AZDO_ORGANIZATION_NAME=$GIT_ORGANIZATION_NAME \
+  AZDO_PROJECT_NAME=$GIT_PROJECT_NAME \
+  AZDO_REPOSITORY_NAME=$GIT_REPOSITORY_NAME \
+  AZDO_PIPELINES_BRANCH_NAME=$azdo_pipelines_branch_name \
+  AZDO_POLICIES_BRANCH_NAME=$azdo_policies_branch_name \
+  bash -c "./scripts/deploy_azdo_pipelines.sh"
+else
+  echo "[Warning] Skipping Azure DevOps pipelines deployment as those are deployed using the user context."
+fi

e2e_samples/fabric_dataops_sample/devops/scripts/build_adls_files.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+set -o errexit
+
+config_template_file="config/application.cfg.template"
+config_file="config/application.cfg"
+
+function get_workspace_id() {
+    access_token=$1
+    workspace_name=$2
+
+    echo "Get workspace ID for $workspace_name/$access_token"
+
+    url="https://api.fabric.microsoft.com/v1/workspaces"
+    response=$(curl -s -H "Authorization: Bearer $access_token" $url)
+
+    echo "[Info] Workspaces: $response"
+
+    workspace_id=$(echo $response | jq -r --arg workspace_name "$workspace_name" '.value[] | select(.displayName == $workspace_name) | .id')
+
+    if [[ -z "$workspace_id" ]]; then
+        echo "[Info] Workspace $workspace_name not found." >&2
+        return 1
+    fi
+
+    export FABRIC_WORKSPACE_ID=$workspace_id
+    echo "Workspace ID: $workspace_id"
+    return 0
+}
+
+function get_lakehouse_id() {
+    access_token=$1
+    workspace_id=$2
+    lakehouse_name=$3
+
+    url="https://api.fabric.microsoft.com/v1/workspaces/$workspace_id/items"
+    response=$(curl -s -H "Authorization: Bearer $access_token" $url)
+
+    lakehouse_id=$(echo $response | jq -r --arg lakehouse_name "$lakehouse_name" '.value[] | select(.displayName == $lakehouse_name and .type == "Lakehouse") | .id')
+
+    if [[ -z "$lakehouse_id" ]]; then
+        echo "[Info] Lakehouse $lakehouse_name not found." >&2
+        return 1
+    fi
+
+    export FABRIC_LAKEHOUSE_ID=$lakehouse_id
+    echo "Lakehouse ID: $lakehouse_id"
+    return 0
+}
+
+# Create a configuration file from the template file
+function create_config_file() {
+    workspace_name=$1
+    workspace_id=$2
+    lakehouse_name=$3
+    lakehouse_id=$4
+    keyvault_name=$5
+    adls_shortcut_name=$6
+    config_file=$7
+
+    echo "Create configuration to config/application.cfg."
+
+    # Read the config template file config/application.cfg.template and replace the placeholders with the actual values
+    sed -e "s/<workspace-name>/$workspace_name/g" \
+        -e "s/<workspace-id>/$workspace_id/g" \
+        -e "s/<lakehouse-name>/$lakehouse_name/g" \
+        -e "s/<lakehouse-id>/$lakehouse_id/g" \
+        -e "s/<keyvault-name>/$keyvault_name/g" \
+        -e "s/<adls-shortcut-name>/$adls_shortcut_name/g" \
+        $config_template_file > $config_file
+
+    echo "Configuration file created at $config_file."
+}
+
+get_workspace_id \
+    "$FABRIC_BEARER_TOKEN"  \
+    "$FABRIC_WORKSPACE_NAME" || { echo "Failed to get workspace_id"; exit 1; }
+
+get_lakehouse_id  \
+    "$FABRIC_BEARER_TOKEN" \
+    "$FABRIC_WORKSPACE_ID" \
+    "$FABRIC_LAKEHOUSE_NAME" || { echo "Failed to get lakehouse_id"; exit 1; }
+
+create_config_file \
+    "$FABRIC_WORKSPACE_NAME" \
+    "$FABRIC_WORKSPACE_ID" \
+    "$FABRIC_LAKEHOUSE_NAME" \
+    "$FABRIC_LAKEHOUSE_ID" \
+    "$KEY_VAULT_NAME" \
+    "$FABRIC_ADLS_SHORTCUT_NAME" \
+    "$config_file"
+
+mkdir -p adls/config/
+cp config/application.cfg adls/config/
+cp config/lakehouse_ddls.yaml adls/config/
+
+mkdir -p adls/reference/
+cp -r data/seed/* adls/reference