Notebook Migration — Databricks to Microsoft Fabric¶

# Databricks cell 1
%python
df = spark.read.format("delta").load("/mnt/data/customers")

# Databricks cell 2
%sql
SELECT * FROM customers WHERE state = 'VA'

# Databricks cell 3
%scala
val df = spark.read.format("delta").load("/mnt/data/customers")

# Databricks cell 4
%r
library(SparkR)
df <- read.df("/mnt/data/customers", source = "delta")

# Fabric cell 1 (language: PySpark)
df = spark.read.format("delta").load("Tables/customers")

# Fabric cell 2 (language: Spark SQL)
SELECT * FROM lakehouse1.customers WHERE state = 'VA'

# Fabric cell 3 -- Scala is NOT available in Fabric
# Rewrite in PySpark:
df = spark.read.format("delta").load("Tables/customers")

# Fabric cell 4 (language: SparkR)
library(SparkR)
df <- read.df("Tables/customers", source = "delta")

%run ./shared/utilities
%run ./config/settings

# Option 1: mssparkutils.notebook.run() -- executes in a new session
result = mssparkutils.notebook.run("shared/utilities", timeout_seconds=120)

# Option 2: mssparkutils.notebook.runMultiple() -- parallel execution
mssparkutils.notebook.runMultiple(["shared/utilities", "config/settings"])

# Option 3: %run is also supported in Fabric (same syntax)
%run shared/utilities

%pip install pandas==2.1.0 scikit-learn==1.3.0
%conda install -c conda-forge lightgbm

# %pip works the same way
%pip install pandas==2.1.0 scikit-learn==1.3.0

# %conda is NOT supported in Fabric
# Use %pip or Fabric environments for conda-managed packages

# For persistent library management, use Fabric environments (see section 6)

# Databricks
df = spark.read.format("delta").load("/mnt/bronze/customers")
df.write.format("delta").mode("overwrite").save("/mnt/silver/customers_clean")

# Fabric (relative paths within default Lakehouse)
df = spark.read.format("delta").load("Tables/bronze_customers")
df.write.format("delta").mode("overwrite").saveAsTable("silver_customers_clean")

# Databricks
secret = dbutils.secrets.get(scope="my-scope", key="storage-key")

# Fabric -- uses Azure Key Vault via mssparkutils
secret = mssparkutils.credentials.getSecret(
    "https://my-keyvault.vault.azure.net/",
    "storage-key"
)

# Fabric -- using linked Key Vault
secret = mssparkutils.credentials.getSecret(
    "my-keyvault",       # linked service name
    "storage-key"        # secret name
)

# Databricks -- create widgets
dbutils.widgets.text("start_date", "2024-01-01", "Start Date")
dbutils.widgets.dropdown("environment", "dev", ["dev", "staging", "prod"])
start_date = dbutils.widgets.get("start_date")
environment = dbutils.widgets.get("environment")

# Fabric -- receive parameters (passed from Data Pipeline or notebook.run)
# Parameters are automatically available as variables when called from:
#   mssparkutils.notebook.run("notebook", params={"start_date": "2024-01-01"})
# Or from a Data Pipeline notebook activity with parameters

# In the notebook, use mssparkutils to get parameters:
start_date = mssparkutils.notebook.getParam("start_date", "2024-01-01")
environment = mssparkutils.notebook.getParam("environment", "dev")

# Databricks
dbutils.notebook.exit("SUCCESS: processed 1000 rows")

# Fabric
mssparkutils.notebook.exit("SUCCESS: processed 1000 rows")

# Databricks -- spark session is pre-configured with cluster settings
# Custom config:
spark.conf.set("spark.sql.shuffle.partitions", "200")
spark.conf.set("spark.databricks.delta.optimizeWrite.enabled", "true")  # Databricks-specific

# Fabric -- spark session is pre-configured with capacity settings
# Custom config:
spark.conf.set("spark.sql.shuffle.partitions", "200")
# Databricks-specific configs (spark.databricks.*) are NOT available
# Fabric equivalent for optimize write:
spark.conf.set("spark.microsoft.delta.optimizeWrite.enabled", "true")   # Fabric-specific

# Databricks
spark.sql("OPTIMIZE my_table ZORDER BY (customer_id)")
spark.sql("VACUUM my_table RETAIN 168 HOURS")

# Fabric -- auto-optimization handles most cases
# V-Order is applied automatically on write
# Manual OPTIMIZE is available but rarely needed:
spark.sql("OPTIMIZE my_table")  # ZORDER syntax is supported
spark.sql("VACUUM my_table RETAIN 168 HOURS")  # Same syntax

# Databricks (Unity Catalog)
df = spark.table("catalog.schema.customers")
df.write.mode("overwrite").saveAsTable("catalog.schema.customers_clean")

# Fabric (Lakehouse)
df = spark.table("lakehouse1.customers")
df.write.mode("overwrite").saveAsTable("customers_clean")
# Or with explicit lakehouse reference:
df.write.mode("overwrite").saveAsTable("lakehouse1.customers_clean")

# In a notebook, you can also use inline installation:
%pip install great-expectations==0.18.0

# For production, use Fabric environments (admin portal):
# Workspace > Environments > New Environment > Add Libraries

# Connect to Fabric Lakehouse SQL endpoint from local Python
import pyodbc

connection_string = (
    "Driver={ODBC Driver 18 for SQL Server};"
    "Server=<workspace-guid>.datawarehouse.fabric.microsoft.com;"
    "Database=<lakehouse-name>;"
    "Authentication=ActiveDirectoryInteractive;"
    "Encrypt=yes;"
    "TrustServerCertificate=no;"
)

conn = pyodbc.connect(connection_string)
cursor = conn.cursor()
cursor.execute("SELECT * FROM customers LIMIT 10")
rows = cursor.fetchall()

"""
databricks_to_fabric_notebook.py
Converts Databricks notebook source to Fabric-compatible format.
Handles: dbutils -> mssparkutils, path translation, magic commands.
Does NOT handle: Scala code, complex init scripts, Databricks-specific Spark configs.
"""

import re
import json
from pathlib import Path

def convert_notebook(source_path: str, output_path: str):
    """Convert a Databricks .py notebook export to Fabric-compatible format."""

    with open(source_path, "r") as f:
        content = f.read()

    # Replace dbutils.fs with mssparkutils.fs
    content = content.replace("dbutils.fs.", "mssparkutils.fs.")

    # Replace dbutils.secrets with mssparkutils.credentials
    content = re.sub(
        r'dbutils\.secrets\.get\(scope="([^"]+)",\s*key="([^"]+)"\)',
        r'mssparkutils.credentials.getSecret("key-vault-name", "\2")',
        content
    )

    # Replace dbutils.widgets.get with mssparkutils equivalent
    content = re.sub(
        r'dbutils\.widgets\.get\("([^"]+)"\)',
        r'mssparkutils.notebook.getParam("\1", "")',
        content
    )

    # Replace dbutils.notebook.exit
    content = content.replace("dbutils.notebook.exit", "mssparkutils.notebook.exit")

    # Replace dbutils.notebook.run
    content = content.replace("dbutils.notebook.run", "mssparkutils.notebook.run")

    # Replace /mnt/ paths with Fabric-style paths
    content = re.sub(
        r'/mnt/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_/-]+)',
        r'Files/\1/\2',
        content
    )

    # Replace Databricks-specific Spark configs
    content = content.replace(
        "spark.databricks.delta.optimizeWrite.enabled",
        "spark.microsoft.delta.optimizeWrite.enabled"
    )

    # Remove widget creation (handled differently in Fabric)
    content = re.sub(
        r'dbutils\.widgets\.(text|dropdown|combobox|multiselect)\([^)]+\)\n?',
        '# Widget removed -- use notebook parameters instead\n',
        content
    )

    # Flag Scala cells for manual rewrite
    content = re.sub(
        r'# MAGIC %scala',
        '# TODO: Rewrite Scala cell in PySpark (Fabric does not support Scala notebooks)',
        content
    )

    with open(output_path, "w") as f:
        f.write(content)

    print(f"Converted: {source_path} -> {output_path}")
    print("IMPORTANT: Manual review required for Scala code, complex configs, and path patterns.")