🚀 Tutorial 1: Environment Setup and Prerequisites¶

# Login to Azure (will open browser for authentication)
az login

# List available subscriptions
az account list --output table

# Set the subscription you want to use for tutorials
az account set --subscription "your-subscription-id-here"

# Verify current subscription
az account show --output table

# Check Synapse Analytics quota
az synapse quota list --location "East US" --output table

# Check compute quotas (for Spark pools)
az vm list-usage --location "East US" --output table | Where-Object {$_.Name.LocalizedValue -like "*Standard D*"}

# Check storage account quotas
az storage account check-quota --output table

# Register required providers
$providers = @(
    'Microsoft.Synapse',
    'Microsoft.Storage', 
    'Microsoft.EventHub',
    'Microsoft.StreamAnalytics',
    'Microsoft.DataFactory',
    'Microsoft.PowerBI',
    'Microsoft.Purview'
)

foreach ($provider in $providers) {
    Write-Host "Registering $provider..." -ForegroundColor Green
    az provider register --namespace $provider
}

# Check registration status
foreach ($provider in $providers) {
    az provider show --namespace $provider --query "registrationState" --output tsv
}

# Install via PowerShell
Invoke-WebRequest -Uri https://aka.ms/installazurecliwindows -OutFile .\AzureCLI.msi
Start-Process msiexec.exe -ArgumentList '/i AzureCLI.msi /quiet' -Wait

# Install Azure PowerShell module
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
Install-Module -Name Az -Repository PSGallery -Force -AllowClobber

# Verify installation
Get-Module Az -ListAvailable | Select-Object Name, Version

# Install VS Code extensions via command line
code --install-extension ms-vscode.azure-account
code --install-extension ms-azuretools.vscode-azureresourcegroups  
code --install-extension ms-python.python
code --install-extension ms-toolsai.jupyter
code --install-extension ms-mssql.mssql
code --install-extension redhat.vscode-yaml

# Install Git via winget
winget install --id Git.Git -e --source winget

# Install Python 3.9+ (Windows)
winget install python.python.3

# Install required Python packages
pip install azure-cli azure-storage-blob azure-identity pandas numpy jupyter

# Create a budget for tutorial expenses
az consumption budget create \
  --budget-name "synapse-tutorial-budget" \
  --amount 100 \
  --time-grain Monthly \
  --start-date "$(Get-Date -Format 'yyyy-MM-01')" \
  --end-date "$(Get-Date -Year $((Get-Date).Year + 1) -Format 'yyyy-MM-01')" \
  --resource-group-filter "synapse-tutorial-rg"

# Create action group for budget alerts
az monitor action-group create \
  --name "budget-alerts" \
  --resource-group "synapse-tutorial-rg" \
  --short-name "budget" \
  --email-receiver name="admin" email="your-email@example.com"

# Set default tags for tutorial resources
$defaultTags = @{
    'Project' = 'SynapseTutorial'
    'Environment' = 'Learning'
    'Owner' = 'YourName'
    'CostCenter' = 'Training'
    'AutoShutdown' = 'Enabled'
}

# Save tags for later use in tutorials
$defaultTags | ConvertTo-Json | Out-File "tutorial-tags.json"

# Interactive region selection
Write-Host "🌍 Azure Region Selection for Synapse Tutorials" -ForegroundColor Cyan
Write-Host ""

# Display available regions with Synapse support
$synapseRegions = @(
    @{Name="East US"; Code="eastus"; Compliance="HIPAA, SOC2"; Pricing="Baseline"},
    @{Name="East US 2"; Code="eastus2"; Compliance="HIPAA, SOC2"; Pricing="Baseline"},
    @{Name="West Europe"; Code="westeurope"; Compliance="GDPR, ISO 27001"; Pricing="+10%"},
    @{Name="North Europe"; Code="northeurope"; Compliance="GDPR, ISO 27001"; Pricing="+10%"},
    @{Name="UK South"; Code="uksouth"; Compliance="GDPR, UK DPA"; Pricing="+15%"},
    @{Name="Southeast Asia"; Code="southeastasia"; Compliance="PDPA"; Pricing="+15%"},
    @{Name="Australia East"; Code="australiaeast"; Compliance="IRAP, Privacy Act"; Pricing="+20%"}
)

Write-Host "Available Azure Regions for Synapse Analytics:" -ForegroundColor Green
$synapseRegions | ForEach-Object -Begin {$i=1} -Process {
    Write-Host "$i. $($_.Name) ($($_.Code))"
    Write-Host "   Compliance: $($_.Compliance)"
    Write-Host "   Relative Cost: $($_.Pricing)"
    Write-Host ""
    $i++
}

# Prompt for region selection
$regionChoice = Read-Host "Select region number (default: 1 - East US)"
if ([string]::IsNullOrWhiteSpace($regionChoice)) {
    $regionChoice = 1
}

$selectedRegion = $synapseRegions[$regionChoice - 1]
$location = $selectedRegion.Code
$locationName = $selectedRegion.Name

Write-Host "Selected Region: $locationName ($location)" -ForegroundColor Green

# Validate data residency requirements
function Test-DataResidency {
    param(
        [string]$Region,
        [string[]]$ComplianceRequirements
    )

    $regionCompliance = @{
        "eastus" = @("HIPAA", "SOC2", "ISO 27001")
        "westeurope" = @("GDPR", "ISO 27001", "ISO 27018")
        "uksouth" = @("GDPR", "UK DPA", "ISO 27001")
        "southeastasia" = @("PDPA", "ISO 27001")
    }

    $supportedCompliance = $regionCompliance[$Region]
    $allRequirementsMet = $true

    foreach ($requirement in $ComplianceRequirements) {
        if ($supportedCompliance -notcontains $requirement) {
            Write-Warning "Region $Region does not support $requirement compliance"
            $allRequirementsMet = $false
        }
    }

    return $allRequirementsMet
}

# Example: Validate GDPR compliance for EU deployment
if ($location -like "*europe*" -or $location -eq "uksouth") {
    $isCompliant = Test-DataResidency -Region $location -ComplianceRequirements @("GDPR")
    if ($isCompliant) {
        Write-Host "✅ Region meets GDPR compliance requirements" -ForegroundColor Green
    }
}

# Test latency to Azure region endpoints
function Test-RegionLatency {
    param([string]$Region)

    $endpoints = @{
        "eastus" = "eastus-management.azure.com"
        "westeurope" = "westeurope-management.azure.com"
        "southeastasia" = "southeastasia-management.azure.com"
    }

    $endpoint = $endpoints[$Region]
    if ($endpoint) {
        Write-Host "Testing latency to $Region..." -ForegroundColor Yellow
        $latency = Test-Connection -ComputerName $endpoint -Count 4 |
            Measure-Object -Property ResponseTime -Average

        $avgLatency = [math]::Round($latency.Average, 2)
        Write-Host "Average latency: $avgLatency ms" -ForegroundColor Cyan

        if ($avgLatency -lt 50) {
            Write-Host "✅ Excellent latency for interactive workloads" -ForegroundColor Green
        } elseif ($avgLatency -lt 150) {
            Write-Host "⚠️ Acceptable latency, consider closer region for real-time workloads" -ForegroundColor Yellow
        } else {
            Write-Host "❌ High latency, strongly recommend selecting closer region" -ForegroundColor Red
        }
    }
}

Test-RegionLatency -Region $location

# Calculate estimated costs for selected region
function Get-RegionalCostEstimate {
    param(
        [string]$Region,
        [int]$SparkHoursPerMonth = 100,
        [int]$StorageGB = 1000
    )

    # Pricing multipliers (relative to East US baseline)
    $pricingMultipliers = @{
        "eastus" = 1.0
        "eastus2" = 1.0
        "westeurope" = 1.10
        "northeurope" = 1.10
        "uksouth" = 1.15
        "southeastasia" = 1.15
        "australiaeast" = 1.20
    }

    # Base pricing (East US)
    $baseSparkCostPerHour = 0.50
    $baseStorageCostPerGB = 0.02

    $multiplier = $pricingMultipliers[$Region]
    if (-not $multiplier) { $multiplier = 1.0 }

    $sparkCost = $SparkHoursPerMonth * $baseSparkCostPerHour * $multiplier
    $storageCost = $StorageGB * $baseStorageCostPerGB * $multiplier
    $totalCost = $sparkCost + $storageCost

    Write-Host "💰 Estimated Monthly Costs for $Region" -ForegroundColor Cyan
    Write-Host "   Spark Pool (100 hours): `$$([math]::Round($sparkCost, 2))"
    Write-Host "   Storage (1TB): `$$([math]::Round($storageCost, 2))"
    Write-Host "   Total Estimate: `$$([math]::Round($totalCost, 2))"
    Write-Host ""

    return $totalCost
}

Get-RegionalCostEstimate -Region $location

# Define naming convention variables (using selected region)
$subscriptionId = (az account show --query "id" --output tsv)
$locationShort = switch ($location) {
    "eastus" { "eus" }
    "eastus2" { "eus2" }
    "westeurope" { "weu" }
    "northeurope" { "neu" }
    "uksouth" { "uks" }
    "southeastasia" { "sea" }
    "australiaeast" { "aue" }
    default { $location.Substring(0, 3) }
}
$environment = "dev"
$project = "syntut"  # synapse tutorial abbreviated

# Generate unique suffix
$uniqueSuffix = [System.Guid]::NewGuid().ToString().Substring(0, 8)

# Create naming convention object
$namingConvention = @{
    'Location' = $location
    'LocationName' = $locationName
    'ResourceGroupName' = "rg-$project-$environment-$locationShort"
    'SynapseWorkspace' = "syn-$project-$environment-$uniqueSuffix"
    'StorageAccount' = "st$project$environment$uniqueSuffix"  # Storage names must be globally unique
    'KeyVault' = "kv-$project-$environment-$uniqueSuffix"
    'SqlPool' = "sql-$project-$environment"
    'SparkPool' = "spark-$project-$environment"
}

# Save naming convention for use in subsequent tutorials
$namingConvention | ConvertTo-Json | Out-File "naming-convention.json"
Write-Host "✅ Naming convention saved to naming-convention.json" -ForegroundColor Green
Write-Host "   Region: $($namingConvention.LocationName) ($($namingConvention.Location))" -ForegroundColor Cyan

# Function to validate Azure resource names
function Test-AzureResourceName {
    param(
        [string]$Name,
        [string]$ResourceType
    )

    switch ($ResourceType) {
        'StorageAccount' {
            if ($Name.Length -gt 24 -or $Name -notmatch '^[a-z0-9]+$') {
                Write-Warning "Storage account name '$Name' is invalid. Must be 3-24 characters, lowercase letters and numbers only."
                return $false
            }
        }
        'SynapseWorkspace' {
            if ($Name.Length -gt 50 -or $Name -notmatch '^[a-zA-Z0-9-]+$') {
                Write-Warning "Synapse workspace name '$Name' is invalid. Must be 1-50 characters, letters, numbers, and hyphens only."
                return $false
            }
        }
    }
    return $true
}

# Validate our naming convention
$namingConvention = Get-Content "naming-convention.json" | ConvertFrom-Json
Test-AzureResourceName -Name $namingConvention.StorageAccount -ResourceType "StorageAccount"
Test-AzureResourceName -Name $namingConvention.SynapseWorkspace -ResourceType "SynapseWorkspace"

# Create comprehensive validation script
$validationScript = @'
# Azure Synapse Tutorial Environment Validation Script
Write-Host "🔍 Validating Azure Synapse Tutorial Environment..." -ForegroundColor Cyan

$errors = @()
$warnings = @()

# Test 1: Azure CLI Authentication
Write-Host "Testing Azure CLI authentication..." -ForegroundColor Yellow
try {
    $account = az account show --query "id" --output tsv
    if ($account) {
        Write-Host "✅ Azure CLI authenticated successfully" -ForegroundColor Green
    } else {
        $errors += "❌ Azure CLI not authenticated"
    }
} catch {
    $errors += "❌ Azure CLI authentication failed: $_"
}

# Test 2: Required PowerShell Modules
Write-Host "Checking PowerShell modules..." -ForegroundColor Yellow
$requiredModules = @('Az.Accounts', 'Az.Synapse', 'Az.Storage', 'Az.Resources')
foreach ($module in $requiredModules) {
    if (Get-Module -ListAvailable -Name $module) {
        Write-Host "✅ Module $module installed" -ForegroundColor Green
    } else {
        $warnings += "⚠️ Module $module not installed (will install when needed)"
    }
}

# Test 3: Tool Availability  
Write-Host "Checking tool availability..." -ForegroundColor Yellow
$tools = @{
    'az' = 'Azure CLI'
    'git' = 'Git'
    'python' = 'Python'
    'code' = 'VS Code (optional)'
}

foreach ($tool in $tools.GetEnumerator()) {
    try {
        $null = Get-Command $tool.Key -ErrorAction Stop
        Write-Host "✅ $($tool.Value) available" -ForegroundColor Green
    } catch {
        if ($tool.Key -eq 'code') {
            $warnings += "⚠️ $($tool.Value) not found (optional)"
        } else {
            $errors += "❌ $($tool.Value) not found"
        }
    }
}

# Test 4: Resource Provider Registration
Write-Host "Checking resource provider registration..." -ForegroundColor Yellow
$providers = @('Microsoft.Synapse', 'Microsoft.Storage', 'Microsoft.EventHub')
foreach ($provider in $providers) {
    $status = az provider show --namespace $provider --query "registrationState" --output tsv
    if ($status -eq 'Registered') {
        Write-Host "✅ $provider registered" -ForegroundColor Green
    } else {
        $warnings += "⚠️ $provider not registered (status: $status)"
    }
}

# Test 5: Subscription Quotas
Write-Host "Checking subscription quotas..." -ForegroundColor Yellow
try {
    $location = "East US"
    $quotas = az vm list-usage --location $location --output json | ConvertFrom-Json
    $coresQuota = $quotas | Where-Object {$_.name.localizedValue -eq "Total Regional Cores"}

    if ($coresQuota.currentValue -lt ($coresQuota.limit - 20)) {
        Write-Host "✅ Sufficient compute quota available" -ForegroundColor Green
    } else {
        $warnings += "⚠️ Limited compute quota remaining"
    }
} catch {
    $warnings += "⚠️ Unable to check compute quotas"
}

# Summary
Write-Host "`n🎯 Validation Summary:" -ForegroundColor Cyan
if ($errors.Count -eq 0) {
    Write-Host "✅ Environment validation passed! Ready to proceed with tutorials." -ForegroundColor Green
    if ($warnings.Count -gt 0) {
        Write-Host "`nWarnings:" -ForegroundColor Yellow
        $warnings | ForEach-Object { Write-Host $_ -ForegroundColor Yellow }
    }
} else {
    Write-Host "❌ Environment validation failed. Please resolve the following issues:" -ForegroundColor Red
    $errors | ForEach-Object { Write-Host $_ -ForegroundColor Red }
    if ($warnings.Count -gt 0) {
        Write-Host "`nAdditional warnings:" -ForegroundColor Yellow
        $warnings | ForEach-Object { Write-Host $_ -ForegroundColor Yellow }
    }
}
'@

# Save and execute validation script
$validationScript | Out-File "validate-environment.ps1"
.\validate-environment.ps1

# Load naming convention
$naming = Get-Content "naming-convention.json" | ConvertFrom-Json

# Create resource group
az group create \
  --name $naming.ResourceGroupName \
  --location "East US" \
  --tags Project=SynapseTutorial Environment=Learning

Write-Host "✅ Resource group '$($naming.ResourceGroupName)' created successfully" -ForegroundColor Green

# Test connectivity to key Azure endpoints
$endpoints = @(
    'https://management.azure.com',
    'https://login.microsoftonline.com',
    'https://graph.microsoft.com'
)

foreach ($endpoint in $endpoints) {
    try {
        $response = Invoke-WebRequest -Uri $endpoint -Method Head -TimeoutSec 10
        Write-Host "✅ Connectivity to $endpoint: OK" -ForegroundColor Green
    } catch {
        Write-Host "❌ Connectivity to $endpoint: Failed" -ForegroundColor Red
    }
}

# Tutorial cost estimation (approximate monthly costs)
$costEstimate = @{
    'Synapse Workspace' = '$0 (free tier available)'
    'Spark Pool (Medium)' = '$50-100/month (auto-pause enabled)'
    'Dedicated SQL Pool' = '$1,000+/month (DW100c, pause when not in use)'
    'Storage (ADLS Gen2)' = '$5-15/month'
    'Event Hubs' = '$10-20/month'
    'Power BI Premium' = '$20/user/month (Pro tier sufficient for tutorials)'
}

Write-Host "💰 Estimated Monthly Costs for Tutorial Environment:" -ForegroundColor Cyan
$costEstimate.GetEnumerator() | Sort-Object Key | ForEach-Object {
    Write-Host "  $($_.Key): $($_.Value)" -ForegroundColor White
}

# Create configuration for auto-scaling and pausing
$autoScaleConfig = @{
    'SparkPool' = @{
        'AutoPause' = @{
            'Enabled' = $true
            'DelayInMinutes' = 15
        }
        'AutoScale' = @{
            'Enabled' = $true
            'MinNodeCount' = 3
            'MaxNodeCount' = 10
        }
    }
    'DedicatedSQLPool' = @{
        'AutoPause' = @{
            'Enabled' = $true
            'DelayInMinutes' = 60
        }
        'AutoScale' = @{
            'Enabled' = $false  # Manual scaling recommended
            'TargetServiceLevelObjective' = 'DW100c'
        }
    }
}

# Save configuration for use in later tutorials
$autoScaleConfig | ConvertTo-Json -Depth 3 | Out-File "auto-scale-config.json"

# Create directory structure for tutorial assets
$assetDirs = @(
    'sample-data',
    'scripts',
    'notebooks',
    'sql-scripts',
    'config-files'
)

foreach ($dir in $assetDirs) {
    New-Item -ItemType Directory -Path $dir -Force
    Write-Host "✅ Created directory: $dir" -ForegroundColor Green
}

# Download sample datasets (placeholder URLs - replace with actual assets)
$sampleData = @{
    'customer-data.csv' = 'https://raw.githubusercontent.com/your-org/synapse-tutorials/main/data/customers.csv'
    'transaction-data.json' = 'https://raw.githubusercontent.com/your-org/synapse-tutorials/main/data/transactions.json'
    'product-catalog.parquet' = 'https://raw.githubusercontent.com/your-org/synapse-tutorials/main/data/products.parquet'
}

foreach ($dataset in $sampleData.GetEnumerator()) {
    try {
        Invoke-WebRequest -Uri $dataset.Value -OutFile "sample-data\$($dataset.Key)"
        Write-Host "✅ Downloaded: $($dataset.Key)" -ForegroundColor Green
    } catch {
        Write-Host "⚠️ Could not download $($dataset.Key) - will create synthetic data in next tutorial" -ForegroundColor Yellow
    }
}

# Create environment summary document
$environmentSummary = @"
# Azure Synapse Tutorial Environment Summary

## Setup Date: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')

## Azure Configuration
- Subscription ID: $(az account show --query "id" --output tsv)
- Default Location: East US
- Resource Group: $($naming.ResourceGroupName)

## Naming Convention
$(Get-Content "naming-convention.json" | ConvertFrom-Json | ConvertTo-Json)

## Tools Installed
- Azure CLI: $(az version --query '\"azure-cli\"' --output tsv)
- Azure PowerShell: $(if (Get-Module Az.Accounts -ListAvailable) { 'Installed' } else { 'Not Installed' })
- Git: $(if (Get-Command git -ErrorAction SilentlyContinue) { git --version } else { 'Not Available' })
- Python: $(if (Get-Command python -ErrorAction SilentlyContinue) { python --version } else { 'Not Available' })

## Cost Management
- Budget Name: synapse-tutorial-budget
- Budget Amount: $100/month
- Auto-pause Configuration: Enabled
- Resource Tags: Applied for cost tracking

## Next Steps
1. Proceed to Tutorial 2: Synapse Workspace Basics
2. Review cost monitoring dashboard regularly
3. Remember to pause/stop resources when not in use

## Support
- Documentation: https://docs.microsoft.com/azure/synapse-analytics/
- Community: https://techcommunity.microsoft.com/t5/azure-synapse-analytics/bd-p/AzureSynapseAnalytics
- Issues: Create issues in the tutorial repository
"@

$environmentSummary | Out-File "environment-summary.md" -Encoding UTF8
Write-Host "✅ Environment summary saved to environment-summary.md" -ForegroundColor Green

# Run final validation
Write-Host "🔍 Final Environment Check..." -ForegroundColor Cyan

# Check resource group exists
$rgExists = az group exists --name (Get-Content "naming-convention.json" | ConvertFrom-Json).ResourceGroupName
if ($rgExists -eq 'true') {
    Write-Host "✅ Resource group exists and ready" -ForegroundColor Green
} else {
    Write-Host "❌ Resource group not found" -ForegroundColor Red
}

# Check configuration files exist
$configFiles = @('naming-convention.json', 'auto-scale-config.json', 'environment-summary.md')
foreach ($file in $configFiles) {
    if (Test-Path $file) {
        Write-Host "✅ Configuration file exists: $file" -ForegroundColor Green
    } else {
        Write-Host "⚠️ Configuration file missing: $file" -ForegroundColor Yellow
    }
}

Write-Host "`n🎯 Environment setup complete! Ready for Tutorial 2." -ForegroundColor Green

# Clear cached credentials and retry
az account clear
az login --use-device-code

# Verify your role assignments
az role assignment list --assignee $(az account show --query user.name --output tsv) --output table

# Force re-registration
az provider register --namespace Microsoft.Synapse --wait

# Set execution policy for current user
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser