1. Understanding Data Quality Issues

Common data quality problems you'll encounter:

• Missing values (NaN, null, empty strings)
• Duplicate records
• Inconsistent formatting
• Outliers and anomalies

Identifying Issues

import pandas as pd
import numpy as np

# Load your dataset
df = pd.read_csv('data.csv')

# Basic data exploration
print("Dataset shape:", df.shape)
print("\nData types:")
print(df.dtypes)
print("\nMissing values:")
print(df.isnull().sum())
print("\nDuplicate rows:", df.duplicated().sum())
              

2. Handling Missing Values

Strategies for dealing with missing data:

Detection and Analysis

# Visualize missing data patterns
import matplotlib.pyplot as plt
import seaborn as sns

# Missing data heatmap
plt.figure(figsize=(10, 6))
sns.heatmap(df.isnull(), yticklabels=False, cbar=True, cmap='viridis')
plt.title('Missing Data Pattern')
plt.show()
              

Handling Strategies

# Drop rows with any missing values
df_drop_rows = df.dropna()

# Drop columns with too many missing values
df_drop_cols = df.dropna(axis=1, thresh=len(df)*0.7)

# Fill missing values
df_filled = df.fillna({
    'numeric_column': df['numeric_column'].mean(),
    'categorical_column': df['categorical_column'].mode()[0]
})

# Forward fill and backward fill
df_ffill = df.fillna(method='ffill')
df_bfill = df.fillna(method='bfill')
              

3. Handling Duplicates

Remove or manage duplicate records:

# Find duplicate rows
duplicates = df.duplicated()
print(f"Number of duplicates: {duplicates.sum()}")

# Remove duplicates
df_no_duplicates = df.drop_duplicates()

# Remove duplicates based on specific columns
df_unique = df.drop_duplicates(subset=['column1', 'column2'])

# Keep last occurrence of duplicates
df_keep_last = df.drop_duplicates(keep='last')
              

4. Data Type Conversion

Ensure proper data types for analysis:

# Convert data types
df['date_column'] = pd.to_datetime(df['date_column'])
df['category_column'] = df['category_column'].astype('category')
df['numeric_column'] = pd.to_numeric(df['numeric_column'], errors='coerce')

# String cleaning
df['text_column'] = df['text_column'].str.strip().str.lower()
              

5. Outlier Detection and Treatment

Identify and handle outliers:

# Statistical method (IQR)
Q1 = df['numeric_column'].quantile(0.25)
Q3 = df['numeric_column'].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR

# Identify outliers
outliers = df[(df['numeric_column'] < lower_bound) | 
              (df['numeric_column'] > upper_bound)]

# Remove outliers
df_clean = df[(df['numeric_column'] >= lower_bound) & 
              (df['numeric_column'] <= upper_bound)]
              

1. Problem Definition and Data Preparation

Start with clear problem definition and clean data:

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder

# Load and explore data
df = pd.read_csv('dataset.csv')
print("Dataset shape:", df.shape)
print("\nTarget variable distribution:")
print(df['target'].value_counts())

# Prepare features and target
X = df.drop('target', axis=1)
y = df['target']

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
              

2. Feature Engineering and Scaling

Transform features for better model performance:

# Handle categorical variables
le = LabelEncoder()
for col in X_train.select_dtypes(include=['object']).columns:
    X_train[col] = le.fit_transform(X_train[col])
    X_test[col] = le.transform(X_test[col])

# Scale numerical features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
              

3. Model Selection and Training

Try different algorithms and compare performance:

from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, classification_report

# Initialize models
models = {
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'Logistic Regression': LogisticRegression(random_state=42),
    'SVM': SVC(random_state=42)
}

# Train and evaluate models
for name, model in models.items():
    # Train model
    if name == 'Logistic Regression' or name == 'SVM':
        model.fit(X_train_scaled, y_train)
        y_pred = model.predict(X_test_scaled)
    else:
        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
    
    # Evaluate
    accuracy = accuracy_score(y_test, y_pred)
    print(f"\n{name} Accuracy: {accuracy:.4f}")
    print(f"Classification Report:")
    print(classification_report(y_test, y_pred))
              

4. Model Validation and Hyperparameter Tuning

Use cross-validation and grid search for optimal parameters:

from sklearn.model_selection import cross_val_score, GridSearchCV

# Cross-validation
rf_model = RandomForestClassifier(random_state=42)
cv_scores = cross_val_score(rf_model, X_train, y_train, cv=5)
print(f"Cross-validation scores: {cv_scores}")
print(f"Mean CV score: {cv_scores.mean():.4f} (+/- {cv_scores.std() * 2:.4f})")

# Hyperparameter tuning
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [10, 20, None],
    'min_samples_split': [2, 5, 10]
}

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid, cv=5, scoring='accuracy'
)
grid_search.fit(X_train, y_train)

print(f"Best parameters: {grid_search.best_params_}")
print(f"Best CV score: {grid_search.best_score_:.4f}")
              

5. Model Evaluation and Interpretation

Evaluate final model and understand predictions:

from sklearn.metrics import confusion_matrix, roc_auc_score
import matplotlib.pyplot as plt
import seaborn as sns

# Final model predictions
best_model = grid_search.best_estimator_
y_pred_final = best_model.predict(X_test)
y_pred_proba = best_model.predict_proba(X_test)[:, 1]

# Confusion Matrix
cm = confusion_matrix(y_test, y_pred_final)
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix')
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()

# Feature importance
feature_importance = pd.DataFrame({
    'feature': X.columns,
    'importance': best_model.feature_importances_
}).sort_values('importance', ascending=False)

print("Top 10 Most Important Features:")
print(feature_importance.head(10))