SAS to Azure: Migration Best Practices¶

import pandas as pd
import numpy as np
from scipy import stats

def reconcile_outputs(sas_output, python_output, key_columns,
                      numeric_tolerance=0.001, report_path=None):
    """Compare SAS and Python outputs for validation.

    Args:
        sas_output: DataFrame from SAS (exported to CSV or Delta)
        python_output: DataFrame from Python
        key_columns: List of columns to join on
        numeric_tolerance: Maximum relative difference for numeric columns
        report_path: Optional path to save HTML report

    Returns:
        Dictionary with validation results
    """
    results = {'passed': True, 'details': []}

    # 1. Row count comparison
    sas_rows = len(sas_output)
    python_rows = len(python_output)
    row_match = sas_rows == python_rows
    results['details'].append({
        'check': 'Row count',
        'sas': sas_rows,
        'python': python_rows,
        'passed': row_match
    })
    if not row_match:
        results['passed'] = False

    # 2. Column comparison
    sas_cols = set(sas_output.columns)
    python_cols = set(python_output.columns)
    missing_in_python = sas_cols - python_cols
    extra_in_python = python_cols - sas_cols
    results['details'].append({
        'check': 'Column match',
        'missing_in_python': list(missing_in_python),
        'extra_in_python': list(extra_in_python),
        'passed': len(missing_in_python) == 0
    })

    # 3. Merge on key columns for row-level comparison
    common_cols = sas_cols & python_cols - set(key_columns)
    merged = pd.merge(sas_output, python_output, on=key_columns,
                       suffixes=('_sas', '_python'), how='outer',
                       indicator=True)

    # Check for unmatched rows
    sas_only = (merged['_merge'] == 'left_only').sum()
    python_only = (merged['_merge'] == 'right_only').sum()
    both = (merged['_merge'] == 'both').sum()
    results['details'].append({
        'check': 'Row matching',
        'matched': both,
        'sas_only': sas_only,
        'python_only': python_only,
        'passed': sas_only == 0 and python_only == 0
    })

    # 4. Numeric column comparison
    for col in common_cols:
        sas_col = f'{col}_sas'
        py_col = f'{col}_python'
        if sas_col not in merged.columns:
            continue

        if merged[sas_col].dtype in ['float64', 'int64', 'float32', 'int32']:
            matched = merged[merged['_merge'] == 'both']
            sas_vals = matched[sas_col].dropna()
            py_vals = matched[py_col].dropna()

            if len(sas_vals) == 0:
                continue

            mae = np.mean(np.abs(sas_vals.values - py_vals.values))
            max_diff = np.max(np.abs(sas_vals.values - py_vals.values))
            sas_mean = sas_vals.mean()
            rel_diff = mae / abs(sas_mean) if sas_mean != 0 else mae

            col_passed = rel_diff < numeric_tolerance
            results['details'].append({
                'check': f'Numeric: {col}',
                'sas_mean': round(sas_mean, 6),
                'python_mean': round(py_vals.mean(), 6),
                'mae': round(mae, 8),
                'max_diff': round(max_diff, 8),
                'relative_diff': round(rel_diff, 8),
                'passed': col_passed
            })
            if not col_passed:
                results['passed'] = False

    # Generate report
    if report_path:
        _generate_html_report(results, report_path)

    return results

# Example: analyze SAS program dependencies
# Look for: %include, libname references, proc append targets,
# data set references across programs

def analyze_sas_dependencies(sas_program_dir):
    """Analyze SAS program dependencies to determine migration order.

    Programs with no dependencies should be migrated first.
    Programs depended upon by others should be migrated before dependents.
    """
    import re
    from pathlib import Path

    programs = {}
    for sas_file in Path(sas_program_dir).glob('*.sas'):
        content = sas_file.read_text(encoding='latin1')
        deps = set()

        # Find %include references
        for match in re.finditer(r"%include\s+['\"]?(.+?)['\"]?\s*;", content, re.I):
            deps.add(match.group(1))

        # Find dataset references (simplified)
        for match in re.finditer(r"set\s+(\w+\.\w+)", content, re.I):
            deps.add(match.group(1))

        programs[sas_file.stem] = {
            'file': str(sas_file),
            'dependencies': deps,
            'lines': len(content.splitlines())
        }

    return programs

# Standard header for migrated programs
"""
Program: survey_analysis.py
Purpose: Quarterly employee survey engagement analysis
Migrated from: /sas/programs/survey/quarterly_analysis.sas
Original author: J. Smith (SAS)
Migration author: J. Smith (Python)
Migration date: 2026-05-15
Validation: Level 3 (full row-level), passed 2026-05-28
Schedule: Monthly, 1st business day (ADF trigger)

Change log:
  2026-05-15  Initial migration from SAS
  2026-06-01  Added Copilot integration for NLQ
"""

# Standard monitoring pattern for migrated programs
import logging
import time
from datetime import datetime

logger = logging.getLogger(__name__)

def run_with_monitoring(program_name, func, *args, **kwargs):
    """Wrapper for migrated SAS programs with standard monitoring."""
    start_time = time.time()
    logger.info(f"Starting {program_name} at {datetime.now().isoformat()}")

    try:
        result = func(*args, **kwargs)
        elapsed = time.time() - start_time
        logger.info(f"Completed {program_name} in {elapsed:.1f}s")

        # Log to Azure Monitor
        # (replaces SAS log monitoring)
        return result

    except Exception as e:
        elapsed = time.time() - start_time
        logger.error(f"Failed {program_name} after {elapsed:.1f}s: {str(e)}")
        raise