SAS to Azure: Complete Feature Mapping¶

import pandas as pd
import numpy as np

df = spark.sql("SELECT * FROM raw.transactions").toPandas()

# Filter
df = df[df['amount'] > 0].copy()

# Missing value imputation
df['category'] = df['category'].fillna('UNKNOWN')

# Date calculations
df['quarter'] = df['transaction_date'].dt.quarter
df['fiscal_year'] = (df['transaction_date'] + pd.DateOffset(months=3)).dt.year

# Transformations
df['amount_log'] = np.log(df['amount'] + 1)

# Conditional logic (replaces IF/THEN/ELSE)
conditions = [
    df['state'].isin(['NY', 'NJ', 'CT']),
    df['state'].isin(['CA', 'OR', 'WA'])
]
choices = ['Northeast', 'West']
df['region'] = np.select(conditions, choices, default='Other')

from pyspark.sql import functions as F
from pyspark.sql.functions import when, col, quarter, year, log, add_months

df = spark.table("raw.transactions")

df_clean = (df
    .filter(col("amount") > 0)
    .withColumn("category", when(col("category").isNull(), "UNKNOWN").otherwise(col("category")))
    .withColumn("quarter", quarter("transaction_date"))
    .withColumn("fiscal_year", year(add_months("transaction_date", 3)))
    .withColumn("amount_log", log(col("amount") + 1))
    .withColumn("region",
        when(col("state").isin("NY", "NJ", "CT"), "Northeast")
        .when(col("state").isin("CA", "OR", "WA"), "West")
        .otherwise("Other"))
)

df_sorted = (df
    .sort_values(['region', 'amount'], ascending=[True, False])
    .drop_duplicates(subset=['region'], keep='first'))

df_wide = df.pivot_table(
    index='region',
    columns='quarter',
    values='total_amount',
    aggfunc='sum'
).reset_index()

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score

def run_analysis(df, target, predictors, output_table):
    """Replaces %macro run_analysis."""
    X = df[predictors]
    y = df[target]

    model = LogisticRegression(max_iter=1000)
    model.fit(X, y)

    df['pred_prob'] = model.predict_proba(X)[:, 1]
    spark.createDataFrame(df).write.mode("overwrite").saveAsTable(output_table)
    return model

model = run_analysis(
    df=loans_df,
    target='default_flag',
    predictors=['credit_score', 'debt_ratio', 'loan_amount'],
    output_table='work.scored_loans'
)

# Fabric lakehouses are referenced by catalog.schema.table
# No LIBNAME equivalent needed - tables are in Unity Catalog

# Read from Oracle (via Spark JDBC)
df = spark.read.format("jdbc").options(
    url="jdbc:oracle:thin:@db-server:1521/PROD",
    dbtable="schema.table",
    driver="oracle.jdbc.OracleDriver"
).load()

# Read from lakehouse
df_staging = spark.table("staging.bronze.raw_transactions")
df_gold = spark.table("gold.fact_transactions")

import scipy.stats as stats

summary = df.groupby('region')[['amount', 'credit_score']].agg(
    ['count', 'mean', 'std', 'min', 'max', 'median']
)

# Add confidence intervals (CLM equivalent)
def confidence_interval(series, confidence=0.95):
    n = len(series)
    mean = series.mean()
    se = stats.sem(series)
    ci = stats.t.interval(confidence, df=n-1, loc=mean, scale=se)
    return pd.Series({'lower_cl': ci[0], 'upper_cl': ci[1]})

ci = df.groupby('region')['amount'].apply(confidence_interval)

# Simple frequency table
freq = df['category'].value_counts()

# Cross-tabulation with chi-square
ct = pd.crosstab(df['region'], df['risk_level'])
chi2, p_value, dof, expected = stats.chi2_contingency(ct)
print(f"Chi-square: {chi2:.4f}, p-value: {p_value:.4f}, df: {dof}")

import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

# Descriptive statistics
desc = df['amount'].describe(percentiles=[.01, .05, .10, .25, .50, .75, .90, .95, .99])

# Normality tests
shapiro_stat, shapiro_p = stats.shapiro(df['amount'].sample(5000))
ks_stat, ks_p = stats.kstest(df['amount'], 'norm',
                              args=(df['amount'].mean(), df['amount'].std()))

# Histogram with normal overlay
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
sns.histplot(df['amount'], kde=True, stat='density', ax=axes[0])
stats.probplot(df['amount'], dist="norm", plot=axes[1])
plt.tight_layout()
plt.show()

import statsmodels.api as sm
from statsmodels.stats.outliers_influence import variance_inflation_factor

X = df[['credit_score', 'debt_ratio', 'loan_term']]
X = sm.add_constant(X)
y = df['amount']

model = sm.OLS(y, X).fit()
print(model.summary())  # R-squared, coefficients, p-values, F-statistic

# VIF (collinearity diagnostics)
vif_data = pd.DataFrame({
    'Feature': X.columns[1:],
    'VIF': [variance_inflation_factor(X.values, i+1) for i in range(X.shape[1]-1)]
})

# Durbin-Watson
from statsmodels.stats.stattools import durbin_watson
dw = durbin_watson(model.resid)

# Influence diagnostics (Cook's D, leverage)
influence = model.get_influence()
cooks_d = influence.cooks_distance[0]
leverage = influence.hat_matrix_diag

import statsmodels.api as sm
from sklearn.metrics import roc_auc_score, roc_curve, classification_report

# statsmodels for detailed output (concordance, Hosmer-Lemeshow)
X = pd.get_dummies(df[['credit_grade', 'credit_score', 'debt_ratio']],
                   drop_first=True)
X = sm.add_constant(X)
y = df['default_flag']

logit_model = sm.Logit(y, X).fit()
print(logit_model.summary())  # Coefficients, Wald tests, pseudo R-squared

# Concordance (c-statistic = AUC)
pred_prob = logit_model.predict(X)
auc = roc_auc_score(y, pred_prob)
print(f"Concordance (c-statistic): {auc:.4f}")

# Hosmer-Lemeshow test
from statsmodels.stats.diagnostic import acorr_ljungbox
# Custom implementation for HL test
def hosmer_lemeshow(y_true, y_pred, n_groups=10):
    data = pd.DataFrame({'y': y_true, 'p': y_pred})
    data['group'] = pd.qcut(data['p'], n_groups, duplicates='drop')
    obs = data.groupby('group')['y'].agg(['sum', 'count'])
    exp = data.groupby('group')['p'].agg(['sum', 'count'])
    hl_stat = (((obs['sum'] - exp['sum'])**2) /
               (exp['count'] * exp['sum']/exp['count'] *
                (1 - exp['sum']/exp['count']))).sum()
    p_val = 1 - stats.chi2.cdf(hl_stat, n_groups - 2)
    return hl_stat, p_val

hl_stat, hl_p = hosmer_lemeshow(y, pred_prob)

# Classification table at multiple thresholds
for threshold in np.arange(0.1, 1.0, 0.1):
    y_pred = (pred_prob >= threshold).astype(int)
    print(f"\nThreshold: {threshold:.1f}")
    print(classification_report(y, y_pred))

import statsmodels.formula.api as smf

model = smf.mixedlm(
    "outcome ~ treatment + age + baseline_score",
    data=df,
    groups=df["center"],
    re_formula="~1"  # Random intercept
).fit()

print(model.summary())

from statsmodels.tsa.arima.model import ARIMA
import pmdarima as pm

# Manual ARIMA
model = ARIMA(df['revenue'], order=(1, 1, 1))
results = model.fit()
forecast = results.forecast(steps=12)

# Auto ARIMA (equivalent to SAS identify + estimate)
auto_model = pm.auto_arima(
    df['revenue'],
    seasonal=True, m=12,
    stepwise=True,
    trace=True
)
forecast = auto_model.predict(n_periods=12)

from samplics.estimation import TaylorEstimator

estimator = TaylorEstimator("mean")
estimator.estimate(
    y=df['income'],
    samp_weight=df['sampling_weight'],
    stratum=df['region'],
    psu=df['psu']
)
print(estimator.to_dataframe())

import matplotlib.pyplot as plt
from IPython.display import display, HTML

# Tables
summary = df.groupby('region')['amount'].agg(['count', 'mean', 'std', 'sum'])
display(summary.style.format("{:,.2f}").set_caption("Quarterly Analysis Report"))

# Charts
fig, ax = plt.subplots(figsize=(10, 6))
df.groupby('region')['amount'].sum().plot(kind='bar', ax=ax)
ax.set_title('Total Amount by Region')
plt.tight_layout()

# Export to PDF
# Use nbconvert or Power BI paginated reports for formatted PDF output

import seaborn as sns

g = sns.FacetGrid(df, col='region', col_wrap=2, height=4)
g.map_dataframe(sns.scatterplot, x='credit_score', y='amount', hue='risk_level')
g.map_dataframe(sns.regplot, x='credit_score', y='amount',
                scatter=False, lowess=True, color='black')
g.add_legend()
plt.tight_layout()