🔄 Tutorial 12: CI/CD and DevOps for Microsoft Fabric¶

# Azure DevOps CI/CD Pipeline for Microsoft Fabric
# Triggers on pushes to main and feature branches

trigger:
  branches:
    include:
      - main
      - feature/*
      - release/*
  paths:
    include:
      - fabric/**
      - scripts/**

pr:
  branches:
    include:
      - main
  paths:
    include:
      - fabric/**

variables:
  - group: fabric-variables  # Variable group with secrets
  - name: fabricWorkspaceId
    value: 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
  - name: deploymentPipelineId
    value: 'yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy'

stages:
  # ============================================
  # Stage 1: Build and Validate
  # ============================================
  - stage: Build
    displayName: 'Build & Validate'
    jobs:
      - job: ValidateFabricItems
        displayName: 'Validate Fabric Items'
        pool:
          vmImage: 'ubuntu-latest'
        steps:
          - checkout: self
            fetchDepth: 0

          - task: PowerShell@2
            displayName: 'Install Fabric Tools'
            inputs:
              targetType: 'inline'
              script: |
                Install-Module -Name MicrosoftPowerBIMgmt -Force -Scope CurrentUser
                Install-Module -Name Az.Accounts -Force -Scope CurrentUser

          - task: PowerShell@2
            displayName: 'Validate Notebook Syntax'
            inputs:
              filePath: 'scripts/validate-notebooks.ps1'
              arguments: '-Path fabric/dev'

          - task: PowerShell@2
            displayName: 'Validate Pipeline JSON'
            inputs:
              filePath: 'scripts/validate-pipelines.ps1'
              arguments: '-Path fabric/dev'

          - task: PowerShell@2
            displayName: 'Validate Semantic Model'
            inputs:
              filePath: 'scripts/validate-semantic-model.ps1'
              arguments: '-Path fabric/dev'

          - task: PublishBuildArtifacts@1
            displayName: 'Publish Fabric Artifacts'
            inputs:
              PathtoPublish: 'fabric'
              ArtifactName: 'fabric-items'

  # ============================================
  # Stage 2: Deploy to Test
  # ============================================
  - stage: DeployTest
    displayName: 'Deploy to Test'
    dependsOn: Build
    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
    jobs:
      - deployment: DeployToTest
        displayName: 'Deploy to Test Environment'
        pool:
          vmImage: 'ubuntu-latest'
        environment: 'fabric-test'
        strategy:
          runOnce:
            deploy:
              steps:
                - task: PowerShell@2
                  displayName: 'Authenticate to Fabric'
                  inputs:
                    targetType: 'inline'
                    script: |
                      $securePassword = ConvertTo-SecureString "$(servicePrincipalSecret)" -AsPlainText -Force
                      $credential = New-Object System.Management.Automation.PSCredential("$(servicePrincipalId)", $securePassword)
                      Connect-PowerBIServiceAccount -ServicePrincipal -Credential $credential -TenantId "$(tenantId)"

                - task: PowerShell@2
                  displayName: 'Trigger Deployment Pipeline - Dev to Test'
                  inputs:
                    filePath: 'scripts/deploy-fabric.ps1'
                    arguments: >
                      -PipelineId "$(deploymentPipelineId)"
                      -SourceStage "Development"
                      -TargetStage "Test"
                      -WaitForCompletion $true

                - task: PowerShell@2
                  displayName: 'Run Integration Tests'
                  inputs:
                    filePath: 'tests/integration/run-tests.ps1'
                    arguments: '-Environment Test'

  # ============================================
  # Stage 3: Deploy to Production
  # ============================================
  - stage: DeployProd
    displayName: 'Deploy to Production'
    dependsOn: DeployTest
    condition: succeeded()
    jobs:
      - deployment: DeployToProduction
        displayName: 'Deploy to Production Environment'
        pool:
          vmImage: 'ubuntu-latest'
        environment: 'fabric-production'  # Requires approval
        strategy:
          runOnce:
            deploy:
              steps:
                - task: PowerShell@2
                  displayName: 'Authenticate to Fabric'
                  inputs:
                    targetType: 'inline'
                    script: |
                      $securePassword = ConvertTo-SecureString "$(servicePrincipalSecret)" -AsPlainText -Force
                      $credential = New-Object System.Management.Automation.PSCredential("$(servicePrincipalId)", $securePassword)
                      Connect-PowerBIServiceAccount -ServicePrincipal -Credential $credential -TenantId "$(tenantId)"

                - task: PowerShell@2
                  displayName: 'Pre-Deployment Backup'
                  inputs:
                    filePath: 'scripts/backup-workspace.ps1'
                    arguments: '-WorkspaceId "$(prodWorkspaceId)"'

                - task: PowerShell@2
                  displayName: 'Trigger Deployment Pipeline - Test to Prod'
                  inputs:
                    filePath: 'scripts/deploy-fabric.ps1'
                    arguments: >
                      -PipelineId "$(deploymentPipelineId)"
                      -SourceStage "Test"
                      -TargetStage "Production"
                      -WaitForCompletion $true

                - task: PowerShell@2
                  displayName: 'Post-Deployment Validation'
                  inputs:
                    filePath: 'scripts/validate-deployment.ps1'
                    arguments: '-WorkspaceId "$(prodWorkspaceId)"'

                - task: PowerShell@2
                  displayName: 'Refresh Semantic Model'
                  inputs:
                    filePath: 'scripts/refresh-semantic-model.ps1'
                    arguments: '-WorkspaceId "$(prodWorkspaceId)" -DatasetName "casino-semantic"'

# GitHub Actions CI/CD Pipeline for Microsoft Fabric

name: Fabric CI/CD

on:
  push:
    branches:
      - main
      - 'feature/**'
    paths:
      - 'fabric/**'
      - 'scripts/**'
      - '.github/workflows/**'
  pull_request:
    branches:
      - main
    paths:
      - 'fabric/**'

env:
  FABRIC_WORKSPACE_ID: ${{ secrets.FABRIC_WORKSPACE_ID }}
  DEPLOYMENT_PIPELINE_ID: ${{ secrets.DEPLOYMENT_PIPELINE_ID }}

jobs:
  # ============================================
  # Job 1: Validate
  # ============================================
  validate:
    name: Validate Fabric Items
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Dependencies
        run: |
          pip install pyspark pytest great-expectations
          pip install azure-identity msal

      - name: Validate Notebook Syntax
        run: |
          python scripts/validate_notebooks.py --path fabric/dev

      - name: Validate Pipeline JSON
        run: |
          python scripts/validate_pipelines.py --path fabric/dev

      - name: Run Unit Tests
        run: |
          pytest tests/unit/ -v --junitxml=test-results.xml

      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-results
          path: test-results.xml

      - name: Upload Fabric Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: fabric-items
          path: fabric/

  # ============================================
  # Job 2: Deploy to Test
  # ============================================
  deploy-test:
    name: Deploy to Test
    needs: validate
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    runs-on: ubuntu-latest
    environment: test
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Download Artifacts
        uses: actions/download-artifact@v4
        with:
          name: fabric-items
          path: fabric/

      - name: Setup PowerShell
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            Install-Module -Name MicrosoftPowerBIMgmt -Force -Scope CurrentUser

      - name: Authenticate to Fabric
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            $securePassword = ConvertTo-SecureString "${{ secrets.SERVICE_PRINCIPAL_SECRET }}" -AsPlainText -Force
            $credential = New-Object System.Management.Automation.PSCredential("${{ secrets.SERVICE_PRINCIPAL_ID }}", $securePassword)
            Connect-PowerBIServiceAccount -ServicePrincipal -Credential $credential -TenantId "${{ secrets.TENANT_ID }}"

      - name: Deploy to Test
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            ./scripts/deploy-fabric.ps1 `
              -PipelineId "${{ secrets.DEPLOYMENT_PIPELINE_ID }}" `
              -SourceStage "Development" `
              -TargetStage "Test" `
              -WaitForCompletion $true

      - name: Run Integration Tests
        run: |
          python tests/integration/run_tests.py --environment test

  # ============================================
  # Job 3: Deploy to Production
  # ============================================
  deploy-production:
    name: Deploy to Production
    needs: deploy-test
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    runs-on: ubuntu-latest
    environment: production  # Requires approval
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Setup PowerShell
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            Install-Module -Name MicrosoftPowerBIMgmt -Force -Scope CurrentUser

      - name: Authenticate to Fabric
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            $securePassword = ConvertTo-SecureString "${{ secrets.SERVICE_PRINCIPAL_SECRET }}" -AsPlainText -Force
            $credential = New-Object System.Management.Automation.PSCredential("${{ secrets.SERVICE_PRINCIPAL_ID }}", $securePassword)
            Connect-PowerBIServiceAccount -ServicePrincipal -Credential $credential -TenantId "${{ secrets.TENANT_ID }}"

      - name: Pre-Deployment Backup
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            ./scripts/backup-workspace.ps1 -WorkspaceId "${{ secrets.PROD_WORKSPACE_ID }}"

      - name: Deploy to Production
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            ./scripts/deploy-fabric.ps1 `
              -PipelineId "${{ secrets.DEPLOYMENT_PIPELINE_ID }}" `
              -SourceStage "Test" `
              -TargetStage "Production" `
              -WaitForCompletion $true

      - name: Post-Deployment Validation
        uses: azure/powershell@v2
        with:
          azPSVersion: 'latest'
          inlineScript: |
            ./scripts/validate-deployment.ps1 -WorkspaceId "${{ secrets.PROD_WORKSPACE_ID }}"

      - name: Notify on Success
        if: success()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "✅ Fabric deployment to Production completed successfully!"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

      - name: Notify on Failure
        if: failure()
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "❌ Fabric deployment to Production failed! Check GitHub Actions for details."
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}