文本分类任务中的特征提取技巧

在大模型训练数据工程中，特征提取是决定模型性能的关键环节。本文分享几个在文本分类任务中实用的特征提取技巧。

1. TF-IDF特征提取

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline

# 构建TF-IDF管道
tfidf_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=10000,
        stop_words='english',
        ngram_range=(1, 2),
        min_df=2,
        max_df=0.8
    )),
])

# 应用到文本数据
X_tfidf = tfidf_pipeline.fit_transform(texts)

2. 字符级N-gram特征

# 启用字符级特征提取
char_tfidf = TfidfVectorizer(
    analyzer='char',
    ngram_range=(2, 5),
    max_features=5000
)

# 结合词级别和字符级别
combined_features = hstack([
    tfidf_pipeline.fit_transform(texts),
    char_tfidf.fit_transform(texts)
])

3. 嵌入层特征提取

import torch
from transformers import AutoTokenizer, AutoModel

# 使用预训练模型提取上下文特征
model_name = 'bert-base-uncased'
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 提取最后一层隐藏状态作为特征
inputs = tokenizer(texts, return_tensors='pt', padding=True, truncation=True)
with torch.no_grad():
    outputs = model(**inputs)
    features = outputs.last_hidden_state.mean(dim=1)  # 平均池化