Pytorch BertModel的使用说明

pip install pytorch-pretrained-bert

import torch
from pytorch_pretrained_bert import BertTokenizer, BertModel, BertFORMaskedLM

tokenizer = BertTokenizer.from_pretrained(--vocab_dir)

text = '[CLS] 武松打老虎 [SEP] 你在哪 [SEP]'
tokenized_text = tokenizer.tokenize(text)
indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
segments_ids = [0, 0, 0, 0, 0, 0, 0,0,0,0, 1,1, 1, 1, 1, 1, 1, 1]
tokens_tensor = torch.tensor([indexed_tokens])
segments_tensors = torch.tensor([segments_ids])

['[', 'cl', '##s', ']', '武', '松', '打', '老', '虎', '[', 'sep', ']', '你', '在', '哪', '[', 'sep', ']']

model = BertModel.from_pretrained(--bert_model)

tokens_tensor = tokens_tensor.cuda()
segments_tensors = segments_tensors.cuda()
model.cuda()

encoded_layers, pooled_output= model(tokens_tensor, segments_tensors)

# -*- encoding: utf-8 -*-
import warnings
warnings.filterwarnings('ignore')
from transformers import BertModel, BertTokenizer, BertConfig
import os
from os.path import dirname, abspath
root_dir = dirname(dirname(dirname(abspath(__file__))))
import torch
# 把预训练的模型从官网下载下来放到目录中
pretrained_path = os.path.join(root_dir, 'pretrained/bert_zh')
# 从文件中加载bert模型
model = BertModel.from_pretrained(pretrained_path)
# 从bert目录中加载词典
tokenizer = BertTokenizer.from_pretrained(pretrained_path)
print(f'vocab size :{tokenizer.vocab_size}')
# 把'[PAD]'编码
print(tokenizer.encode('[PAD]'))
print(tokenizer.encode('[SEP]'))
# 把中文句子编码，默认加入了special tokens了，也就是句子开头加入了[CLS] 句子结尾加入了[SEP]
ids = tokenizer.encode("我是中国人", add_special_tokens=True)
# 从结果中看，101是[CLS]的id，而2769是"我"的id
# [101, 2769, 3221, 704, 1744, 782, 102]
print(ids)
# 把ids解码为中文，默认是没有跳过特殊字符的
print(tokenizer.decode([101, 2769, 3221, 704, 1744, 782, 102], skip_special_tokens=False))
# print(model)
inputs = torch.tensor(ids).unsqueeze(0)
# forward，result是一个tuple，第一个tensor是最后的hidden-state
result = model(torch.tensor(inputs))
# [1, 5, 768]
print(result[0].size())
# [1, 768]
print(result[1].size())
for name, parameter in model.named_parameters():
  # 打印每一层，及每一层的参数
  print(name)
  # 每一层的参数默认都requires_grad=True的，参数是可以学习的
  print(parameter.requires_grad)
  # 如果只想训练第11层transformer的参数的话：
  if '11' in name:
    parameter.requires_grad = True
  else:
    parameter.requires_grad = False
print([p.requires_grad for name, p in model.named_parameters()])

>>> a
tensor([[101,  3,  4, 23, 11,  1, 102,  0,  0,  0]])
>>> notpad = a!=0
>>> notpad
tensor([[ True, True, True, True, True, True, True, False, False, False]])
>>> notcls = a!=101
>>> notcls
tensor([[False, True, True, True, True, True, True, True, True, True]])
>>> notsep = a!=102
>>> notsep
tensor([[ True, True, True, True, True, True, False, True, True, True]])
>>> mask = notpad & notcls & notsep
>>> mask
tensor([[False, True, True, True, True, True, False, False, False, False]])
>>>