总结
1. 没怎么理解代码,生成的文字有些多余的中括号
2.本人电脑显卡不行,跑不了1024层所以不能用其他人模型继续训练
3.从生成的小说看它已经学会了标点符号的使用,双冒号“成对的使用。
4.参数还是需要调整,生成的特定词语一直重复
下载代码
https://github.com/Morizeyao/GPT2-Chinese%20https://github.com/Morizeyao/GPT2-Chinese在根目录（目录\GPT2-Chinese\）下建立文件夹data和model
\GPT2-Chinese\data
\GPT2-Chinese\model
把要训练的小说复制到train.json这里面
train.json（也即->？\GPT2-Chinese\data\train.json），需要注意的是，train.json编码格式严格为UTF-8，并且不带BOM头<-去头咱用的sublime。
vocab
- vocab.txt:字典文件。默认的字典大小为13317，若需要使用自定义字典，需要将confog.json文件中的vocab_size字段设为相应的大小。
安装spyder
pip install -U spyder -i https://pypi.tuna.tsinghua.edu.cn/simple --ignore-installed qtconsole --user
1.安装 pip install -i https://pypi.tuna.tsinghua.edu.cn/simple transformers
2.如果没有就要 pip install -i https://pypi.tuna.tsinghua.edu.cn/simple ipywidgets
由于安装新版的transformers 所以改一下代码
transformers 报错 got_ver is None,重装numpy,
pip uninstall numpy
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple numpy
#model_config = transformers.modeling_gpt2.GPT2Config.from_json_file(args.model_config)
model_config = transformers.models.gpt2.GPT2Config.from_json_file(args.model_config)
if not args.pretrained_model:
#model = transformers.modeling_gpt2.GPT2LMHeadModel(config=model_config)
model = transformers.models.gpt2.GPT2LMHeadModel(config=model_config)
else:
#model = transformers.modeling_gpt2.GPT2LMHeadModel.from_pretrained(args.pretrained_model)
model = transformers.models.gpt2.GPT2LMHeadModel(args.pretrained_model)提示模块'transformers'没有属性'WarmupLinearSchedule'的异常
这是因为在新版本中WarmupLinearSchedule方法已经没有了，可以换为get_linear_schedule_with_warmup方法
# scheduler = transformers.WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps,
# t_total=total_steps)
scheduler = transformers.get_linear_schedule_with_warmup(optimizer, num_warmup_steps =warmup_steps,
num_training_steps =total_steps)然后修改
V1 版本
train_single.py
import transformers import torch print(torch.cuda.current_device()) import torchvision import os import json import random import argparse import numpy as np from datetime import datetime from torch.nn import DataParallel from tqdm import tqdm ''' 如果训练材料是全部堆在一起不分篇章的话用这个文件 ''' def build_files(raw_data_path, tokenized_data_path, full_tokenizer, num_pieces): with open(raw_data_path, 'r', encoding='utf8') as f: print('reading lines') lines = f.readlines() #读取全部行返回一个list lines = [line.replace('\n', ' [SEP] ') for line in lines] # 用[SEP]表示换行, 段落之间使用SEP表示段落结束 single = ''.join(lines) len_single = len(single) if not os.path.exists(tokenized_data_path): os.mkdir(tokenized_data_path) for i in tqdm(range(num_pieces)): single_ids = full_tokenizer.convert_tokens_to_ids( full_tokenizer.tokenize(single[len_single // num_pieces * i: len_single // num_pieces * (i + 1)])) with open(tokenized_data_path + 'tokenized_train_{}.txt'.format(i), 'w') as f: for id in single_ids[:-1]: if(str(id)=='None'): f.write(str(0) + ' ') else: f.write(str(id) + ' ') f.write(str(single_ids[-1])) f.write('\n') print('finish') ''' 检测训练的文件是否已经tokenizer成文件了,跟换训练文件先清空tokenized_data_path目录下所有文件 ''' def is_tokenizer(tokenized_data_path): if not os.path.exists(tokenized_data_path): return False # 获取目录下（包含子目录）的所有文件数 file_nums = sum([len(files) for root,dirs,files in os.walk(tokenized_data_path)]) if file_nums>1: return True else: return False def main(): parser = argparse.ArgumentParser() parser.add_argument('--device', default='0,1,2,3', type=str, required=False, help='设置使用哪些显卡') parser.add_argument('--model_config', default='config/model_config_small_v1.json', type=str, required=False, help='选择模型参数') parser.add_argument('--tokenizer_path', default='cache/vocab_small.txt', type=str, required=False, help='选择词库') parser.add_argument('--raw_data_path', default='data/train.json', type=str, required=False, help='原始训练语料') parser.add_argument('--tokenized_data_path', default='data/tokenized/', type=str, required=False, help='tokenized语料存放位置') parser.add_argument('--raw', action='store_true', help='是否先做tokenize') parser.add_argument('--epochs', default=5, type=int, required=False, help='训练循环') parser.add_argument('--batch_size', default=1, type=int, required=False, help='训练batch size') parser.add_argument('--lr', default=1.5e-4, type=float, required=False, help='学习率') parser.add_argument('--warmup_steps', default=2000, type=int, required=False, help='warm up步数') parser.add_argument('--log_step', default=1, type=int, required=False, help='多少步汇报一次loss') parser.add_argument('--stride', default=768, type=int, required=False, help='训练时取训练数据的窗口步长') parser.add_argument('--gradient_accumulation', default=1, type=int, required=False, help='梯度积累') parser.add_argument('--fp16', action='store_true', help='混合精度') parser.add_argument('--fp16_opt_level', default='O1', type=str, required=False) parser.add_argument('--max_grad_norm', default=1.0, type=float, required=False) parser.add_argument('--num_pieces', default=100, type=int, required=False, help='将训练语料分成多少份') parser.add_argument('--output_dir', default='model/', type=str, required=False, help='模型输出路径') parser.add_argument('--pretrained_model', default='', type=str, required=False, help='模型训练起点路径') parser.add_argument('--segment', action='store_true', help='中文以词为单位') args = parser.parse_args() print('args:\n' + args.__repr__()) if args.segment: from tokenizations import tokenization_bert_word_level as tokenization_bert else: from tokenizations import tokenization_bert os.environ["CUDA_VISIBLE_DEVICES"] = args.device # 此处设置程序使用哪些显卡 # model_config = transformers.modeling_gpt2.GPT2Config.from_json_file(args.model_config) model_config = transformers.models.gpt2.GPT2Config.from_json_file(args.model_config) print('config:\n' + model_config.to_json_string()) n_ctx = model_config.n_ctx full_tokenizer = tokenization_bert.BertTokenizer(vocab_file=args.tokenizer_path) full_tokenizer.max_len = 999999 device = 'cuda' if torch.cuda.is_available() else 'cpu' print('using device:', device) raw_data_path = args.raw_data_path tokenized_data_path = args.tokenized_data_path raw = is_tokenizer(tokenized_data_path) # 选择是否从零开始构建数据集 epochs = args.epochs batch_size = args.batch_size lr = args.lr warmup_steps = args.warmup_steps log_step = args.log_step stride = args.stride gradient_accumulation = args.gradient_accumulation fp16 = args.fp16 # 不支持半精度的显卡请勿打开 fp16_opt_level = args.fp16_opt_level max_grad_norm = args.max_grad_norm num_pieces = args.num_pieces output_dir = args.output_dir if raw == False: print('building files') build_files(raw_data_path=raw_data_path, tokenized_data_path=tokenized_data_path, full_tokenizer=full_tokenizer, num_pieces=num_pieces) print('files built') if not args.pretrained_model: #model = transformers.modeling_gpt2.GPT2LMHeadModel(config=model_config) model = transformers.models.gpt2.GPT2LMHeadModel(config=model_config) else: #model = transformers.modeling_gpt2.GPT2LMHeadModel.from_pretrained(args.pretrained_model) model = transformers.models.gpt2.GPT2LMHeadModel(args.pretrained_model) model.train() model.to(device) print(model) multi_gpu=False full_len = 0 print('calculating total steps') for i in tqdm(range(num_pieces)): with open(tokenized_data_path + 'tokenized_train_{}.txt'.format(i), 'r') as f: full_len += len([int(item) for item in f.read().strip().split()]) print('total steps = {}'.format(total_steps)) optimizer = transformers.AdamW(model.parameters(), lr=lr, correct_bias=True) # scheduler = transformers.WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, # t_total=total_steps) scheduler = transformers.get_linear_schedule_with_warmup(optimizer, num_warmup_steps =warmup_steps, num_training_steps =total_steps) if fp16: try: from apex import amp except ImportError: raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") model, optimizer = amp.initialize(model, optimizer, opt_level=fp16_opt_level) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") model = DataParallel(model) multi_gpu = True print('starting training') running_loss = 0 run_pice =0 for epoch in range(epochs): print('epoch {}'.format(epoch + 1)) now = datetime.now() print('time: {}'.format(now)) x = np.linspace(0, num_pieces - 1, num_pieces, dtype=np.int32) random.shuffle(x) piece_num = 0 for i in x: with open(tokenized_data_path + 'tokenized_train_{}.txt'.format(i), 'r') as f: line = f.read().strip() tokens = line.split() tokens = [int(token) for token in tokens] start_point = 0 samples = [] while start_point < len(tokens) - n_ctx: samples.append(tokens[start_point: start_point + n_ctx]) start_point += stride if start_point < len(tokens): samples.append(tokens[len(tokens)-n_ctx:]) random.shuffle(samples) for step in range(len(samples) // batch_size): # prepare data batch = samples[step * batch_size: (step + 1) * batch_size] batch_labels = [] batch_inputs = [] for ids in batch: int_ids_for_labels = [int(x) for x in ids] int_ids_for_inputs = [int(x) for x in ids] batch_labels.append(int_ids_for_labels) batch_inputs.append(int_ids_for_inputs) batch_labels = torch.tensor(batch_labels).long().to(device) batch_inputs = torch.tensor(batch_inputs).long().to(device) # forward pass outputs = model.forward(input_ids=batch_inputs, labels=batch_labels) loss, logits = outputs[:2] # get loss if multi_gpu: loss = loss.mean() if gradient_accumulation > 1: loss = loss / gradient_accumulation # loss backward if fp16: with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_grad_norm) else: loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm) # optimizer step if (step + 1) % gradient_accumulation == 0: running_loss += loss.item() optimizer.step() optimizer.zero_grad() scheduler.step() if (step + 1) % log_step == 0: print('now time: {}:{}. Step {} of piece {} of epoch {}, loss {}'.format( datetime.now().hour, datetime.now().minute, (step + 1) // gradient_accumulation, piece_num, epoch + 1, running_loss / log_step)) running_loss = 0 if run_pice % 100 == 0: model.save_pretrained(output_dir + 'final_model') run_pice+=1 piece_num += 1 print('saving model for epoch {}'.format(epoch + 1)) if not os.path.exists(output_dir + 'model_epoch{}'.format(epoch + 1)): os.mkdir(output_dir + 'model_epoch{}'.format(epoch + 1)) model_to_save = model.module if hasattr(model, 'module') else model model_to_save.save_pretrained(output_dir + 'model_epoch{}'.format(epoch + 1)) # torch.save(scheduler.state_dict(), output_dir + 'model_epoch{}/scheduler.pt'.format(epoch + 1)) # torch.save(optimizer.state_dict(), output_dir + 'model_epoch{}/optimizer.pt'.format(epoch + 1)) print('epoch {} finished'.format(epoch + 1)) then = datetime.now() print('time: {}'.format(then)) print('time for one epoch: {}'.format(then - now)) print('training finished') if not os.path.exists(output_dir + 'final_model'): os.mkdir(output_dir + 'final_model') model_to_save = model.module if hasattr(model, 'module') else model model_to_save.save_pretrained(output_dir + 'final_model') # torch.save(scheduler.state_dict(), output_dir + 'final_model/scheduler.pt') # torch.save(optimizer.state_dict(), output_dir + 'final_model/optimizer.pt') if __name__ == '__main__': main()generate.py
import torch import torch.nn.functional as F import os import argparse from tqdm import trange from transformers import GPT2LMHeadModel def is_word(word): for item in list(word): if item not in 'qwertyuiopasdfghjklzxcvbnm': return False return True def _is_chinese_char(char): """Checks whether CP is the codepoint of a CJK character.""" # This defines a "chinese character" as anything in the CJK Unicode block: # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) # # Note that the CJK Unicode block is NOT all Japanese and Korean characters, # despite its name. The modern Korean Hangul alphabet is a different block, # as is Japanese Hiragana and Katakana. Those alphabets are used to write # space-separated words, so they are not treated specially and handled # like the all of the other languages. cp = ord(char) if ((cp >= 0x4E00 and cp <= 0x9FFF) or # (cp >= 0x3400 and cp <= 0x4DBF) or # (cp >= 0x20000 and cp <= 0x2A6DF) or # (cp >= 0x2A700 and cp <= 0x2B73F) or # (cp >= 0x2B740 and cp <= 0x2B81F) or # (cp >= 0x2B820 and cp <= 0x2CEAF) or (cp >= 0xF900 and cp <= 0xFAFF) or # (cp >= 0x2F800 and cp <= 0x2FA1F)): # return True return False def top_k_top_p_filtering(logits, top_k=0, top_p=0.0, filter_value=-float('Inf')): """ Filter a distribution of logits using top-k and/or nucleus (top-p) filtering Args: logits: logits distribution shape (vocabulary size) top_k > 0: keep only top k tokens with highest probability (top-k filtering). top_p > 0.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering). Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751) From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317 """ assert logits.dim() == 1 # batch size 1 for now - could be updated for more but the code would be less clear top_k = min(top_k, logits.size(-1)) # Safety check if top_k > 0: # Remove all tokens with a probability less than the last token of the top-k indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] logits[indices_to_remove] = filter_value if top_p > 0.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) # Remove tokens with cumulative probability above the threshold sorted_indices_to_remove = cumulative_probs > top_p # Shift the indices to the right to keep also the first token above the threshold sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = 0 indices_to_remove = sorted_indices[sorted_indices_to_remove] logits[indices_to_remove] = filter_value return logits def sample_sequence(model, context, length, n_ctx, tokenizer, temperature=1.0, top_k=30, top_p=0.0, repitition_penalty=1.0, device='cpu'): context = torch.tensor(context, dtype=torch.long, device=device) context = context.unsqueeze(0) generated = context with torch.no_grad(): for _ in trange(length): inputs = {'input_ids': generated[0][-(n_ctx - 1):].unsqueeze(0)} outputs = model( **inputs) # Note: we could also use 'past' with GPT-2/Transfo-XL/XLNet (cached hidden-states) next_token_logits = outputs[0][0, -1, :] for id in set(generated): next_token_logits[id] /= repitition_penalty next_token_logits = next_token_logits / temperature next_token_logits[tokenizer.convert_tokens_to_ids('[UNK]')] = -float('Inf') filtered_logits = top_k_top_p_filtering(next_token_logits, top_k=top_k, top_p=top_p) next_token = torch.multinomial(F.softmax(filtered_logits, dim=-1), num_samples=1) generated = torch.cat((generated, next_token.unsqueeze(0)), dim=1) return generated.tolist()[0] def fast_sample_sequence(model, context, length, temperature=1.0, top_k=30, top_p=0.0, device='cpu'): inputs = torch.LongTensor(context).view(1, -1).to(device) if len(context) > 1: _, past = model(inputs[:, :-1], None)[:2] prev = inputs[:, -1].view(1, -1) else: past = None prev = inputs generate = [] + context with torch.no_grad(): for i in trange(length): output = model(prev, past=past) output, past = output[:2] output = output[-1].squeeze(0) / temperature filtered_logits = top_k_top_p_filtering(output, top_k=top_k, top_p=top_p) next_token = torch.multinomial(torch.softmax(filtered_logits, dim=-1), num_samples=1) generate.append(next_token.item()) prev = next_token.view(1, 1) return generate # 通过命令行参数--fast_pattern，指定模式 def generate(n_ctx, model, context, length, tokenizer, temperature=1, top_k=0, top_p=0.0, repitition_penalty=1.0, device='cpu', is_fast_pattern=False): if is_fast_pattern: return fast_sample_sequence(model, context, length, temperature=temperature, top_k=top_k, top_p=top_p, device=device) else: return sample_sequence(model, context, length, n_ctx, tokenizer=tokenizer, temperature=temperature, top_k=top_k, top_p=top_p, repitition_penalty=repitition_penalty, device=device) def main(): parser = argparse.ArgumentParser() parser.add_argument('--device', default='0,1,2,3', type=str, required=False, help='生成设备') parser.add_argument('--length', default=256, type=int, required=False, help='生成长度') parser.add_argument('--batch_size', default=1, type=int, required=False, help='生成的batch size') parser.add_argument('--nsamples', default=10, type=int, required=False, help='生成几个样本') parser.add_argument('--temperature', default=1, type=float, required=False, help='生成温度') parser.add_argument('--topk', default=5, type=int, required=False, help='最高几选一') parser.add_argument('--topp', default=0.95, type=float, required=False, help='最高积累概率') parser.add_argument('--model_config', default='config/model_config_small_v1.json', type=str, required=False, help='模型参数') parser.add_argument('--tokenizer_path', default='cache/vocab_small.txt', type=str, required=False, help='词表路径') parser.add_argument('--model_path', default='model/final_model', type=str, required=False, help='模型路径') parser.add_argument('--prefix', default='罗峰', type=str, required=False, help='生成文章的开头') parser.add_argument('--no_wordpiece', action='store_true', help='不做word piece切词') parser.add_argument('--segment', action='store_true', help='中文以词为单位') parser.add_argument('--fast_pattern', action='store_true', help='采用更加快的方式生成文本') parser.add_argument('--save_samples', action='store_true', help='保存产生的样本') parser.add_argument('--save_samples_path', default='.', type=str, required=False, help="保存样本的路径") parser.add_argument('--repetition_penalty', default=1.2, type=float, required=False) # 重复处罚率 args = parser.parse_args() print('args:\n' + args.__repr__()) if args.segment: from tokenizations import tokenization_bert_word_level as tokenization_bert else: from tokenizations import tokenization_bert os.environ["CUDA_VISIBLE_DEVICES"] = args.device # 此处设置程序使用哪些显卡 length = args.length batch_size = args.batch_size nsamples = args.nsamples temperature = args.temperature topk = args.topk topp = args.topp repetition_penalty = args.repetition_penalty device = "cuda" if torch.cuda.is_available() else "cpu" tokenizer = tokenization_bert.BertTokenizer(vocab_file=args.tokenizer_path) model = GPT2LMHeadModel.from_pretrained(args.model_path) model.to(device) model.eval() n_ctx = model.config.n_ctx if length == -1: length = model.config.n_ctx if args.save_samples: if not os.path.exists(args.save_samples_path): os.makedirs(args.save_samples_path) samples_file = open(args.save_samples_path + '/samples.txt', 'w', encoding='utf8') while True: raw_text = args.prefix context_tokens = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(raw_text)) generated = 0 for _ in range(nsamples // batch_size): out = generate( n_ctx=n_ctx, model=model, context=context_tokens, length=length, is_fast_pattern=args.fast_pattern, tokenizer=tokenizer, temperature=temperature, top_k=topk, top_p=topp, repitition_penalty=repetition_penalty, device=device ) for i in range(batch_size): generated += 1 text = tokenizer.convert_ids_to_tokens(out) for i, item in enumerate(text[:-1]): # 确保英文前后有空格 if is_word(item) and is_word(text[i + 1]): text[i] = item + ' ' for i, item in enumerate(text): if item == '[MASK]': text[i] = '' elif item == '[CLS]': text[i] = '\n\n' elif item == '[SEP]': text[i] = '\n' info = "=" * 40 + " SAMPLE " + str(generated) + " " + "=" * 40 + "\n" print(info) text = ''.join(text).replace('##', '').strip() print(text) if args.save_samples: samples_file.write(info) samples_file.write(text) samples_file.write('\n') samples_file.write('=' * 90) samples_file.write('\n' * 2) print("=" * 80) if generated == nsamples: # close file when finish writing. if args.save_samples: samples_file.close() break if __name__ == '__main__': main()
{
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"n_ctx": 512,
"n_embd": 768,
"n_head": 12,
"n_layer": 10,
"n_positions": 512,
"vocab_size": 13317
}
args:
Namespace(batch_size=1, device='0,1,2,3', fast_pattern=False, length=256, model_config='config/model_config_small.json', model_path='model/final_model', no_wordpiece=False, nsamples=10, prefix='罗峰', repetition_penalty=1.2, save_samples=False, save_samples_path='.', segment=False, temperature=1, tokenizer_path='cache/vocab_small.txt', topk=5, topp=0.95)训练一遍后 
文章出处登录后可见！
已经登录？立即刷新
