数据质量评估自动化实现

在大模型训练过程中，数据质量直接影响模型性能。本文将介绍如何自动化评估数据质量，包括缺失值检测、异常值识别和数据分布分析。

核心评估指标

import pandas as pd
import numpy as np
from scipy import stats

def evaluate_data_quality(df):
    quality_report = {}
    
    # 1. 缺失值分析
    missing_data = df.isnull().sum()
    missing_percent = (missing_data / len(df)) * 100
    quality_report['missing_values'] = pd.DataFrame({'count': missing_data, 'percent': missing_percent})
    
    # 2. 异常值检测（基于IQR）
    numeric_columns = df.select_dtypes(include=[np.number]).columns
    outliers = {}
    for col in numeric_columns:
        Q1 = df[col].quantile(0.25)
        Q3 = df[col].quantile(0.75)
        IQR = Q3 - Q1
        lower_bound = Q1 - 1.5 * IQR
        upper_bound = Q3 + 1.5 * IQR
        outlier_count = len(df[(df[col] < lower_bound) | (df[col] > upper_bound)])
        outliers[col] = outlier_count
    
    quality_report['outliers'] = outliers
    
    # 3. 数据分布检查
    distributions = {}
    for col in numeric_columns:
        skewness = stats.skew(df[col].dropna())
        kurtosis = stats.kurtosis(df[col].dropna())
        distributions[col] = {'skewness': skewness, 'kurtosis': kurtosis}
    
    quality_report['distributions'] = distributions
    
    return quality_report

自动化报告生成

def generate_quality_report(df):
    report = evaluate_data_quality(df)
    
    print("=== 数据质量评估报告 ===")
    print(f"数据集形状: {df.shape}")
    
    print("\n缺失值情况:")
    print(report['missing_values'][report['missing_values']['count'] > 0])
    
    print("\n异常值数量:")
    for col, count in report['outliers'].items():
        if count > 0:
            print(f"{col}: {count} 个异常值")
    
    return report