📦 Notebook Resources & Environments - Dependency Management in Fabric¶

# Resource files are accessible at a well-known path
import os

# List all resource files
resource_path = "/lakehouse/default/Files/notebook-resources/"
for f in os.listdir(resource_path):
    print(f)

# Read a JSON configuration file
import json

with open("builtin/config.json") as f:
    config = json.load(f)

print(f"Environment: {config['env']}")
print(f"CTR Threshold: {config['ctr_threshold']}")

# Read a CSV lookup table
import pandas as pd

zones_df = pd.read_csv("builtin/casino_zones.csv")
# Convert to Spark DataFrame for joins
zones_spark = spark.createDataFrame(zones_df)

# Import a Python utility module
import importlib.util

spec = importlib.util.spec_from_file_location("utils", "builtin/utils.py")
utils = importlib.util.module_from_spec(spec)
spec.loader.exec_module(utils)

# Now use functions from the module
result = utils.hash_pii("123-45-6789", salt=config["hash_salt"])

# In Notebook: 01_bronze_slot_telemetry
# =====================================

# Run the shared utilities notebook first
%run bronze_utils

# Now functions from bronze_utils are available
df = read_source_data(spark, "slot_telemetry", processing_date)
df = add_metadata_columns(df)
validate_bronze(df)
write_bronze(df, "slot_telemetry")

# Notebook: bronze_utils
# =======================
# This notebook is %run'd by all bronze layer notebooks.
# It defines shared functions and constants.

from pyspark.sql import functions as F
from datetime import datetime

# ---- CONSTANTS ----
BRONZE_PATH = "abfss://bronze@onelake.dfs.fabric.microsoft.com/lh_bronze.Lakehouse/Tables"
CHECKPOINT_BASE = "abfss://bronze@onelake.dfs.fabric.microsoft.com/lh_bronze.Lakehouse/Files/_checkpoints"

# ---- SHARED FUNCTIONS ----

def read_source_data(spark, table_name, date=None):
    """Read source data from the landing zone."""
    path = f"{BRONZE_PATH}/_landing/{table_name}"
    df = spark.read.format("parquet").load(path)
    if date:
        df = df.filter(F.col("date") == date)
    return df

def add_metadata_columns(df):
    """Add standard metadata columns to bronze DataFrames."""
    return df.withColumns({
        "_ingestion_timestamp": F.current_timestamp(),
        "_source_file": F.input_file_name(),
        "_record_hash": F.sha2(F.concat_ws("|", *df.columns), 256),
    })

def validate_bronze(df, min_rows=1, max_null_rate=0.05):
    """Run standard bronze validations."""
    total = df.count()
    if total < min_rows:
        raise ValueError(f"Expected >= {min_rows} rows, got {total}")

    for col_name in df.columns:
        if not col_name.startswith("_"):
            null_count = df.filter(F.col(col_name).isNull()).count()
            if null_count / total > max_null_rate:
                print(f"WARNING: {col_name} has {null_count/total:.1%} nulls")

def write_bronze(df, table_name, mode="append"):
    """Write DataFrame to bronze Delta table."""
    path = f"{BRONZE_PATH}/{table_name}"
    df.write.format("delta").mode(mode).save(path)
    print(f"Wrote {df.count()} records to {table_name}")

# Pass parameters to the called notebook
%run shared_config {"env": "prod", "agency": "USDA"}

# In shared_config notebook, access via:
env = spark.conf.get("spark.notebook.param.env", "dev")
agency = spark.conf.get("spark.notebook.param.agency", "")

# In notebook settings (gear icon):
# Environment: casino-poc-env

# Or programmatically via metadata:
# The notebook JSON includes:
{
    "environment": {
        "environmentId": "env-id-here",
        "workspaceId": "workspace-id"
    }
}

# Only use for quick testing -- NOT for production
%pip install great-expectations==0.18.0

# For production, add to the Fabric Environment instead

# Generate a lock file from your environment
# (Run in your dev environment)
pip freeze > requirements-lock.txt

# Use the lock file for production environment
# This ensures exact reproducibility

# All casino bronze notebooks start with:
%run bronze_utils

# bronze_utils provides:
# - read_source_data(spark, table, date)
# - add_metadata_columns(df)
# - validate_bronze(df)
# - write_bronze(df, table)
# - hash_pii(value, salt)  # For SSN, card numbers

# Casino-specific resource files:
# - builtin/compliance_thresholds.json
# - builtin/casino_zones.csv
# - builtin/game_type_mappings.json

# Each federal agency notebook includes agency-specific config:

# USDA notebook resources:
# - builtin/usda_api_config.json
# - builtin/crop_categories.csv
# - builtin/state_fips_codes.csv

# NOAA notebook resources:
# - builtin/noaa_station_list.csv
# - builtin/weather_variable_codes.json

# EPA notebook resources:
# - builtin/aqi_breakpoints.json
# - builtin/pollutant_standards.csv