【AI学习路线 04】NLP基础 - 从词向量到Transformer

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

documents = [
    "机器学习是人工智能的核心",
    "深度学习是机器学习的分支",
    "自然语言处理是深度学习的应用"
]

# 词袋模型
bow_vectorizer = CountVectorizer()
bow_matrix = bow_vectorizer.fit_transform(documents)

# TF-IDF
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(documents)

from gensim.models import Word2Vec

# 训练Word2Vec模型
sentences = [
    ['机器学习', '是', '人工智能', '核心'],
    ['深度学习', '是', '机器学习', '分支'],
    ['自然语言处理', '是', '深度学习', '应用']
]

model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)

# 获取词向量
vector = model.wv['机器学习']

# 找相似词
similar = model.wv.most_similar('机器学习', topn=3)

import torch
import torch.nn as nn
import torch.nn.functional as F
import math

class SelfAttention(nn.Module):
    """自注意力实现"""
    
    def __init__(self, embed_size, heads):
        super(SelfAttention, self).__init__()
        self.embed_size = embed_size
        self.heads = heads
        self.head_dim = embed_size // heads
        
        self.values = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.keys = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.queries = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.fc_out = nn.Linear(heads * self.head_dim, embed_size)
    
    def forward(self, values, keys, query, mask):
        N = query.shape[0]
        value_len, key_len, query_len = values.shape[1], keys.shape[1], query.shape[1]
        
        # 分割多头
        values = values.reshape(N, value_len, self.heads, self.head_dim)
        keys = keys.reshape(N, key_len, self.heads, self.head_dim)
        queries = query.reshape(N, query_len, self.heads, self.head_dim)
        
        # 注意力分数
        energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
        if mask is not None:
            energy = energy.masked_fill(mask == 0, float("-1e20"))
        
        attention = torch.softmax(energy / (self.embed_size ** 0.5), dim=3)
        out = torch.einsum("nhql,nlhd->nqhd", [attention, values])
        out = out.reshape(N, query_len, self.heads * self.head_dim)
        
        return self.fc_out(out)

from transformers import BertTokenizer, BertModel, BertForSequenceClassification

# 加载预训练BERT
model_name = 'bert-base-chinese'
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertModel.from_pretrained(model_name)

# 文本编码
text = "自然语言处理是人工智能的重要领域"
inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True)

# 获取BERT输出
with torch.no_grad():
    outputs = model(**inputs)
    
# [CLS] token的表示 (用于分类任务)
cls_embedding = outputs.last_hidden_state[:, 0, :]

from transformers import GPT2LMHeadModel, GPT2Tokenizer

model_name = 'gpt2'
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2LMHeadModel.from_pretrained(model_name)

# 文本生成
input_text = "Natural language processing is"
input_ids = tokenizer.encode(input_text, return_tensors='pt')

output = model.generate(input_ids, max_length=50, temperature=0.7)
generated_text = tokenizer.decode(output[0], skip_special_tokens=True)

from transformers import pipeline

# 情感分析
sentiment_analyzer = pipeline("sentiment-analysis", model="bert-base-chinese")
result = sentiment_analyzer("这个产品非常好用！")

# 问答系统
qa_pipeline = pipeline("question-answering")
result = qa_pipeline(question="什么是机器学习？", 
                     context="机器学习是人工智能的一个分支...")

# 命名实体识别
ner_pipeline = pipeline("ner", aggregation_strategy="simple")
result = ner_pipeline("比尔·盖茨是微软公司的创始人")