Data Processing

1. Vectorized Operations

Pandas and NumPy operations are implemented in C under the hood. Iterating row-by-row in Python is orders of magnitude slower.

import pandas as pd
import numpy as np

# BAD: apply() is essentially a Python for-loop
df['risk_flag'] = df.apply(lambda row: 1 if row['amount'] > 5000 and row['days_late'] > 30 else 0, axis=1)

# GOOD: vectorized boolean mask — 100x+ faster on large DataFrames
df['risk_flag'] = ((df['amount'] > 5000) & (df['days_late'] > 30)).astype(int)

# GOOD: np.where() for conditional assignment
df['risk_label'] = np.where(df['risk_flag'] == 1, 'High', 'Normal')

# GOOD: vectorized string operation (not apply + lambda)
df['state'] = df['address'].str.extract(r',\s*([A-Z]{2})\s*\d{5}')

Performance Comparison

2. Missing Data & Imputation

Types of Missingness

import pandas as pd
from sklearn.impute import SimpleImputer, KNNImputer

# Simple imputation
df['income'].fillna(df['income'].median(), inplace=True)

# Group-based imputation (MAR: income missing more for younger applicants)
df['income'] = df.groupby('age_band')['income'].transform(
    lambda x: x.fillna(x.median())
)

# KNN imputation (uses neighboring rows)
imputer = KNNImputer(n_neighbors=5)
df[['income', 'credit_limit']] = imputer.fit_transform(df[['income', 'credit_limit']])

# Add missingness indicator before imputing (preserves MNAR signal)
df['income_missing'] = df['income'].isna().astype(int)
df['income'].fillna(df['income'].median(), inplace=True)

Outlier Treatment

# Winsorization: clip to 1st and 99th percentile
lower = df['amount'].quantile(0.01)
upper = df['amount'].quantile(0.99)
df['amount_clipped'] = df['amount'].clip(lower, upper)

# Log transformation: stabilize right-skewed distributions
df['log_amount'] = np.log1p(df['amount'])   # log(x + 1) handles zeros

# Z-score outlier detection
z_scores = (df['amount'] - df['amount'].mean()) / df['amount'].std()
df['is_outlier'] = (z_scores.abs() > 3).astype(int)

3. Shaping & Merging

import pandas as pd

# GroupBy with multiple aggregations
summary = df.groupby('customer_id').agg(
    total_spend=('amount', 'sum'),
    txn_count=('amount', 'count'),
    avg_txn=('amount', 'mean'),
    last_txn_date=('txn_date', 'max')
).reset_index()

# Merging — always specify how= explicitly
merged = pd.merge(
    customers,
    transactions,
    on='customer_id',
    how='left'    # keep all customers, even those with no transactions
)
print(f"Before merge: {len(customers)} rows")
print(f"After merge:  {len(merged)} rows")  # should equal len(customers) for left join

# Pivot: wide format
pivot = df.pivot_table(
    index='customer_id',
    columns='month',
    values='spend',
    aggfunc='sum',
    fill_value=0
)

# Melt: long format
long_df = pivot.reset_index().melt(
    id_vars='customer_id',
    var_name='month',
    value_name='spend'
)

4. Feature Engineering

Categorical Encoding

import pandas as pd
from sklearn.preprocessing import StandardScaler

# One-hot encoding
df = pd.get_dummies(df, columns=['product_type'], drop_first=True)

# Target encoding with cross-validation (prevent leakage)
from category_encoders import TargetEncoder
encoder = TargetEncoder(cols=['merchant_category'])
# IMPORTANT: fit only on training data, then transform train and test separately
encoder.fit(X_train['merchant_category'], y_train)
X_train['merchant_category_enc'] = encoder.transform(X_train['merchant_category'])
X_test['merchant_category_enc']  = encoder.transform(X_test['merchant_category'])

# Scaling — fit ONLY on train, transform both
scaler = StandardScaler()
X_train[['income', 'balance']] = scaler.fit_transform(X_train[['income', 'balance']])
X_test[['income', 'balance']]  = scaler.transform(X_test[['income', 'balance']])

# Time-series lag features — sort first!
df = df.sort_values('txn_date')
df['spend_lag_1'] = df.groupby('customer_id')['spend'].shift(1)
df['spend_roll_7'] = df.groupby('customer_id')['spend'].transform(
    lambda x: x.rolling(7, min_periods=1).mean()
)