# -*- coding: utf-8 -*-

import torch
import torch.nn as nn
from tensorflow.keras.preprocessing import sequence
device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu")


class Bert(nn.Module):
    def __init__(self, embedding_dim, target_size):
        super().__init__()
        self.model = torch.hub.load('huggingface/pytorch-transformers', 'model', 'bert-base-chinese')
        self.tokenizer = torch.hub.load('huggingface/pytorch-transformers', 'tokenizer', 'bert-base-chinese')
        self.linear = nn.Linear(embedding_dim, target_size)

    def forward(self, tokens_tensor):
        with torch.no_grad():
            encoded_layers, _ = self.model(tokens_tensor.unsqueeze(0).to(device))
        encoded_layers = encoded_layers[0]
        return self.linear(encoded_layers)

    def get_token_for_single(self, text):
        with torch.no_grad():
            index_tokens = self.tokenizer.encode(text)[1:-1]
            tokens_tensor = torch.tensor([index_tokens])
        return tokens_tensor.squeeze(0)

# -*- coding: utf-8 -*-

# 本段代码构建类BiLSTM, 完成初始化和网络结构的搭建

# 总共3层: 词嵌入层, 双向LSTM层, 全连接线性层

import torch
import torch.nn as nn

class BiLSTM(nn.Module):
    """
    description: BiLSTM 模型定义
    """

    def __init__(self, vocab_size, tag_to_id, input_feature_size, hidden_size,
                 batch_size, sentence_length, num_layers=1, batch_first=True):
        """
        description: 模型初始化
        :param vocab_size:          所有句子包含字符大小
        :param tag_to_id:           标签与 id 对照
        :param input_feature_size:  字嵌入维度( 即LSTM输入层维度 input_size )
        :param hidden_size:         隐藏层向量维度
        :param batch_size:          批训练大小
        :param sentence_length      句子长度
        :param num_layers:          堆叠 LSTM 层数
        :param batch_first:         是否将batch_size放置到矩阵的第一维度
        """

        # 类继承初始化函数

        super(BiLSTM, self).__init__()
        # 设置标签与id对照

        self.tag_to_id = tag_to_id
        # 设置标签大小, 对应BiLSTM最终输出分数矩阵宽度

        self.tag_size = len(tag_to_id)
        # 设定LSTM输入特征大小, 对应词嵌入的维度大小

        self.embedding_size = input_feature_size
        # 设置隐藏层维度, 若为双向时想要得到同样大小的向量, 需要除以2

        self.hidden_size = hidden_size // 2
        # 设置批次大小, 对应每个批次的样本条数, 可以理解为输入张量的第一个维度

        self.batch_size = batch_size
        # 设定句子长度

        self.sentence_length = sentence_length
        # 设定是否将batch_size放置到矩阵的第一维度, 取值True, 或False

        self.batch_first = batch_first
        # 设置网络的LSTM层数

        self.num_layers = num_layers

        # 构建词嵌入层: 字向量, 维度为总单词数量与词嵌入维度

        # 参数: 总体字库的单词数量, 每个字被嵌入的维度

        self.embedding = nn.Embedding(vocab_size, self.embedding_size)

        # 构建双向LSTM层: BiLSTM (参数: input_size      字向量维度(即输入层大小),
        #                               hidden_size     隐藏层维度,
        #                               num_layers      层数,
        #                               bidirectional   是否为双向,
        #                               batch_first     是否批次大小在第一位)

        self.bilstm = nn.LSTM(input_size=input_feature_size,
                              hidden_size=self.hidden_size,
                              num_layers=num_layers,
                              bidirectional=True,
                              batch_first=batch_first)

        # 构建全连接线性层: 将BiLSTM的输出层进行线性变换

        self.linear = nn.Linear(hidden_size, self.tag_size)
	
	# 本函数实现类BiLSTM中的前向计算函数forward()

    def forward(self, sentences_sequence):
        """
        description: 将句子利用BiLSTM进行特征计算，分别经过Embedding->BiLSTM->Linear，
	                 获得发射矩阵（emission scores）
	    :param sentences_sequence: 句子序列对应的编码，
	                               若设定 batch_first 为 True，
	                               则批量输入的 sequence 的 shape 为(batch_size, sequence_length)
	    :return:    返回当前句子特征，转化为 tag_size 的维度的特征
	    """

        # 初始化隐藏状态值

        h0 = torch.randn(self.num_layers * 2, self.batch_size, self.hidden_size)
	    # 初始化单元状态值

        c0 = torch.randn(self.num_layers * 2, self.batch_size, self.hidden_size)
        # 生成字向量， shape 为(batch, sequence_length, input_feature_size)

        # 注：embedding cuda 优化仅支持 SGD 、 SparseAdam

        input_features = self.embedding(sentences_sequence)
        # 将字向量与初始值(隐藏状态 h0 , 单元状态 c0 )传入 LSTM 结构中
        # 输出包含如下内容：
        # 1, 计算的输出特征，shape 为(batch, sentence_length, hidden_size)
        #    顺序为设定 batch_first 为 True 情况, 若未设定则 batch 在第二位
        # 2, 最后得到的隐藏状态 hn ， shape 为(num_layers * num_directions, batch, hidden_size)
        # 3, 最后得到的单元状态 cn ， shape 为(num_layers * num_directions, batch, hidden_size)

        output, (hn, cn) = self.bilstm(input_features, (h0, c0))
        # 将输出特征进行线性变换，转为 shape 为 (batch, sequence_length, tag_size) 大小的特征

        sequence_features = self.linear(output)
        # 输出线性变换为 tag 映射长度的特征

        return sequence_features

# 本函数实现将中文文本映射为数字化的张量
def sentence_map(sentence_list, char_to_id, max_length):
    """
    description: 将句子中的每一个字符映射到码表中
    :param sentence: 待映射句子, 类型为字符串或列表
    :param char_to_id: 码表, 类型为字典, 格式为{"字1": 1, "字2": 2}
    :return: 每一个字对应的编码, 类型为tensor
    """

    # 字符串按照逆序进行排序, 不是必须操作

    sentence_list.sort(key=lambda c:len(c), reverse=True)
    # 定义句子映射列表

    sentence_map_list = []
    for sentence in sentence_list:
        # 生成句子中每个字对应的 id 列表

        sentence_id_list = [char_to_id[c] for c in sentence]
        # 计算所要填充 0 的长度

        padding_list = [0] * (max_length-len(sentence))
        # 组合

        sentence_id_list.extend(padding_list)
        # 将填充后的列表加入句子映射总表中

        sentence_map_list.append(sentence_id_list)
    # 返回句子映射集合, 转为标量

    return torch.tensor(sentence_map_list, dtype=torch.long)

# 参数1:句子集合

sentence_list = [
    "确诊弥漫大b细胞淋巴瘤1年",
    "反复咳嗽、咳痰40年,再发伴气促5天。",
    "生长发育迟缓9年。",
    "右侧小细胞肺癌第三次化疗入院",
    "反复气促、心悸10年,加重伴胸痛3天。",
    "反复胸闷、心悸、气促2多月,加重3天",
    "咳嗽、胸闷1月余, 加重1周",
    "右上肢无力3年, 加重伴肌肉萎缩半年"]

# 参数2:码表与id对照

char_to_id = {"<PAD>":0}

# 参数3:句子长度

SENTENCE_LENGTH = 20

if __name__ == '__main__':
    # 参数1:标签码表对照

    tag_to_id = {"O": 0, "B-dis": 1, "I-dis": 2, "B-sym": 3, "I-sym": 4}
    
    # 参数2:字向量维度

    EMBEDDING_DIM = 200
    
    # 参数3:隐层维度

    HIDDEN_DIM = 100
    
    # 参数4:批次大小

    BATCH_SIZE = 8
    
    # 参数5:句子长度

    SENTENCE_LENGTH = 20
    
    # 参数6:堆叠 LSTM 层数

    NUM_LAYERS = 1
    
    char_to_id = {"<PAD>":0}
    SENTENCE_LENGTH = 20
    for sentence in sentence_list:
        for _char in sentence:
            if _char not in char_to_id:
                char_to_id[_char] = len(char_to_id)
    sentence_sequence = sentence_map(sentence_list, char_to_id, SENTENCE_LENGTH)

    model = BiLSTM(vocab_size=len(char_to_id), tag_to_id=tag_to_id, input_feature_size=EMBEDDING_DIM, \
    hidden_size=HIDDEN_DIM, batch_size=BATCH_SIZE, sentence_length=SENTENCE_LENGTH, num_layers=NUM_LAYERS)

    sentence_features = model(sentence_sequence)
    print("sequence_features:\n", sentence_features)

# -*- coding: utf-8 -*-

# 导入相关包与模块

import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.optim as optim
import Bert
device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu")

# 设置随机种子

torch.manual_seed(1)

# 编写3个辅助函数, 未来在类中会调用这些函数

# 第1个: 求最大值所在的下标

def argmax(vec):
    # 返回最大值对应的下标, 以Python整型返回

    _, idx = torch.max(vec, 1)
    return idx.item()


# 第2个: 文本字符串转换成数字化张量, 以Tensor长整型返回

def prepare_sequence(seq, to_ix):
    idxs = [to_ix[w] for w in seq]
    return torch.tensor(idxs, dtype=torch.long)


# 第3个: 计算log-sum-exp的数值

def log_sum_exp(vec):
    # 先把最大值取出

    max_score = vec[0, argmax(vec)]
    # 使用广播变量将单一数值扩充成一行张量

    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])
    # 一个代码技巧: 这里先减去最大值, 外面再加上最大值, 并不影响数学计算的正确性.

    # 但却提升了代码的健壮性, 大大缓解了最大值溢出错误的问题, 因为计算机进行指数运算很容易溢出

    return max_score + torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))


# 首先设置若干超参数

START_TAG = "<START>"
STOP_TAG = "<STOP>"
EMBEDDING_DIM = 200
HIDDEN_DIM = 100

# 初始化标签字典

tag_to_ix = {"O":0, "B-dis":1, "I-dis":2, "B-sym":3, "I-sym":4, START_TAG:5, STOP_TAG:6}

# 函数sentence_map完成中文文本信息的数字编码, 变成张量

def sentence_map(sentence_list):
    # 初始化数字映射字典

    word_to_ix = {}
    for sentence, tags in sentence_list:
        for word in sentence:
            if word not in word_to_ix:
                word_to_ix[word] = len(word_to_ix)

    return word_to_ix

class BiLSTM_CRF(nn.Module):
    def __init__(self, vocab_size, tag_to_ix, embedding_dim, hidden_dim):
        super(BiLSTM_CRF, self).__init__()
        # 将参数传入类中

        # 词嵌入的维度embedding_dim

        self.embedding_dim = embedding_dim
        # LSTM网络中隐藏层的维度hidden_dim

        self.hidden_dim = hidden_dim
        # 词汇总数vocab_size

        self.vocab_size = vocab_size
        # 标签映射字典tag_to_ix

        self.tag_to_ix = tag_to_ix
        # 标签数量tagset_size, 决定了转移矩阵的维度

        self.tagset_size = len(tag_to_ix)

        # 定义词嵌入层, 输入的两个参数是vocab_size, embedding_dim

        self.word_embeds = nn.Embedding(vocab_size, embedding_dim)
        # 定义BiLSTM层, 设定内部只有一个隐藏层, 并通过参数指定成双向

        # 注意: 因为采用双向网络, 因此隐藏层维度要除以2

        self.lstm = nn.LSTM(embedding_dim, hidden_dim // 2,
                            num_layers=1, bidirectional=True)

        # 定义一个全连接线性层, 用来将BiLSTM网络的输出维度hidden_dim映射到转移矩阵的维度tagset_size 

        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)

        # 定义转移矩阵, 并初始化参数, 采用正态分布的随机初始化赋值

        # 转移矩阵是一个方阵, 维度等于标签数量tagset_size

        # 转移矩阵的任意位置i, j代表从标签j转移到标签i的概率

        self.transitions = nn.Parameter(torch.randn(self.tagset_size, self.tagset_size))

        # 设定两个限制条件, 任意合法的转移不会转移到起始标签, 也不会从结束标签转移出去

        self.transitions.data[tag_to_ix[START_TAG], :] = -10000
        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000

        # 初始化LSTM网络的两个输入张量h0, c0

        self.hidden = self.init_hidden()

        self.bert = Bert.Bert(embedding_dim, len(tag_to_ix))



    def init_hidden(self):
        # 关于LSTM网络的初始输入张量, 三个参数分别代表(num_layers * num_directions, batch_size, hidden_size)

        return (torch.randn(2, 1, self.hidden_dim // 2),
                torch.randn(2, 1, self.hidden_dim // 2))

    def _get_lstm_features(self, sentence):
        # 为一个参数sentence, 代表文本向量化后的输入张量

        # 首先获取一个初始化的隐藏张量h0, c0

        self.hidden = self.init_hidden()
        # 进行词嵌入的处理, 并将一句样本变形为三维张量 

        embeds = self.word_embeds(sentence).view(len(sentence), 1, -1)
        # 将输入张量和初始化张量送入LSTM网络中

        lstm_out, self.hidden = self.lstm(embeds, self.hidden)
        # 再将输出的张量重新变形为二维张量, 以便后续送入全连接层

        lstm_out = lstm_out.view(len(sentence), self.hidden_dim)
        # 进行映射处理, 将LSTM网络的输出维度hidden_dim映射到转移矩阵的维度tagset_size

        lstm_feats = self.hidden2tag(lstm_out)
        # 返回发射矩阵的张量

        return lstm_feats

    # 优化:这里使用bert输出的features

    def _get_bert_features(self, sentence):
        feats = self.bert(sentence)
        return feats

    def _forward_alg(self, feats):
        # 唯一的输入参数feats, 代表经过BiLSTM网络处理后的发射矩阵张量
        # 初始化(1, tagset_size)的二维张量, 全部赋值为-10000

        init_alphas = torch.full((1, self.tagset_size), -10000.).to(device)
        # 只把起始标签START_TAG位置赋值为0, 意思就是只能从起始标签开始进行合法的转移

        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.
    
        # 初始化前向传播张量, 并赋值为上面的初始张量, 也要求最初合法的转移从起始标签START_TAG开始

        forward_var = init_alphas
    
        # 遍历样本句子中的每一个字符

        for feat in feats:
            # 初始化空列表, 存储当前时间步的前向传播张量

            alphas_t = []
            # 遍历所有可能的标签类型, 本项目中是7种可能的标签

            for next_tag in range(self.tagset_size):
                # 初始化发射分数的张量, 无论前一个标签是什么, 经过广播变量后的每一个分量值都相同

                emit_score = feat[next_tag].view(1, -1).expand(1, self.tagset_size)
                # 第i个输入的转移分数代表第i个字符转移到next_tag标签的转移分数

                trans_score = self.transitions[next_tag].view(1, -1)
                # 前向传播张量 + 转移矩阵分数 + 发射矩阵分数, 表示第i个字符转移到next_tag标签的总分值

                next_tag_var = forward_var + trans_score + emit_score
                # 经历log_sum_exp计算后的数值添加进每一个时间步的存储列表中

                alphas_t.append(log_sum_exp(next_tag_var).view(1))
            # 将前向传播张量变形成1行后, 赋值给forward_var, 等待下一个时间步的计算

            forward_var = torch.cat(alphas_t).view(1, -1)
        # 所有时间步for循环结束后, 代表整个句子完成了遍历, 再加上结束标签的转移分数即可

        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]
        # 同理, 对张量计算log_sum_exp的数值

        alpha = log_sum_exp(terminal_var)
        # 最后返回整个样本句子的前向传播分数

        return alpha

    def _score_sentence(self, feats, tags):
        # feats: 代表经过BiLSTM网络处理后的发射矩阵张量

        # tags: 代表真实的标签序列

        # 初始化分数为0

        score = torch.zeros(1).to(device)
        # 模拟NLP的处理流程, 将起始标签START_TAG添加到真实标签序列的最前面

        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long).to(device), tags])
    
        # 遍历发射矩阵张量的每一个时间步, 累加分数 

        for i, feat in enumerate(feats):
            # 每个时间步进行分数的累加, 加的是从第i时间步的标签转移到第i+1时间步的标签分数, 再加上第i+1时间步的发射分数

            score = score + self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]
        # for循环结束后, 代表整个样本句子所有的字符遍历完毕, 最后添加上转移到结束标签STOP_TAG的转移分数即可

        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]
        # 最后返回带标签的真实句子分数

        return score

    def _viterbi_decode(self, feats):
        # feats: 传入的唯一参数是经过BiLSTM网络后得到的发射矩阵张量
        # 初始化回溯指针列表, 里面存放着每一个时间步的解析标签结果

        backpointers = []
    
        # 初始化变量为1行7列的张量, 并赋值全部为-10000

        init_vvars = torch.full((1, self.tagset_size), -10000.).to(device)
        # 只将起始标签位置设置为0, 意思是所有合法的序列都从START_TAG开始

        init_vvars[0][self.tag_to_ix[START_TAG]] = 0
    
        # 初始化前向传播张量, 第i个时间步的forward_var中存放的是第i-1个时间步的维特比变量

        forward_var = init_vvars
        # 遍历样本语句中的每一个时间步, 也就是遍历序列中的每一个字符

        for feat in feats:
            # 初始化当前时间步的回溯指针

            bptrs_t = []
            # 初始化当前时间步的维特比变量

            viterbivars_t = []
    
            # 遍历当前时间步下, 所有的可能标签, 本项目中有7种可能的标签

            for next_tag in range(self.tagset_size):
                # 将上一个时间步标签i对应的维特比变量, 加上从标签i转移到标签next_tag的转移分数, 作为next_tag_var

                # 这里没有加发射矩阵的分数, 是因为发射分数经过广播变量后值都相同, 不影响后面求最大值的操作, 因此不加了

                next_tag_var = forward_var + self.transitions[next_tag]
                # 采用贪心算法, 将最大值作为解析结果

                best_tag_id = argmax(next_tag_var)
                # 将最优解析结果追加存入当前时间步的回溯指针列表中

                bptrs_t.append(best_tag_id)
                # 将维特比变量值存入当前时间步的列表中

                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))
            # 内层for循环结束后, 再加上发射矩阵张量, 作为更新后的前向传播张量, 准备下一个时间步的解析

            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)
            # 将当前时间步的解析指针追加存入总的回溯指针列表中

            backpointers.append(bptrs_t)
    
        # 外层for循环结束后, 说明已经遍历了整个序列, 在最后追加上转移到结束标签的转移分数即可

        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]
        # 求最大值作为最后一步的tag

        best_tag_id = argmax(terminal_var)
        # 采用最佳tag对应的分数值作为整个路径解析的分数值

        path_score = terminal_var[0][best_tag_id]
    
        # 遍历回溯指针列表, 依次解析出最佳路径标签

        # 最后一步的标签首先进入结果列表中

        best_path = [best_tag_id]
        # 逆向遍历回溯指针列表

        for bptrs_t in reversed(backpointers):
            # 每次拿出一个最佳标签, 并用这个最佳标签作为下一步解析的索引

            best_tag_id = bptrs_t[best_tag_id]
            # 依次将每一步解析出来的最佳标签追加进结果列表中

            best_path.append(best_tag_id)
        # 解析的最后一步对应的一定是开始标签START_TAG, 这里删除掉, 因为START_TAG并不是真实的标签序列, 只是人为添加的为程序代码服务的标签

        start = best_path.pop()
        # 确认一下这个标签是开始标签, 属于一个正确性检测代码

        assert start == self.tag_to_ix[START_TAG]
        # 因为是从后向前追加进的结果列表, 所以逆序排列后的结果就是从前向后的真实语序解析结果

        best_path.reverse()
        # 返回解析的语句得分, 还有解析得到的最佳路径

        return path_score, best_path

    # 添加一个计算损失loss的函数

    def neg_log_likelihood(self, sentence, tags):
        # 从BiLSTM网络中得到发射矩阵张量

        # feats = self._get_lstm_features(sentence)

        feats = self._get_bert_features(sentence)
        # 调用手写的训练阶段前向传播函数, 得到前向传播的预测分数值

        forward_score = self._forward_alg(feats)
        # 调用手写的训练阶段句子分值计算函数, 计算真实的句子分数值, 里面有真实的标签序列

        gold_score = self._score_sentence(feats, tags)
        # 预测分数 - 真实分数, 这个差值就是损失值loss 

        return forward_score - gold_score
    
    
    # 添加前向传播函数, 这里面的forward函数不在训练阶段使用, 反倒是在预测阶段才使用

    def forward(self, sentence):
        # 从BiLSTM网络中得到发射矩阵张量

        # lstm_feats = self._get_lstm_features(sentence)

        lstm_feats = self._get_bert_features(sentence)
    
        # 利用发射矩阵张量, 通过维特比算法直接解码得到最佳路径

        score, tag_seq = self._viterbi_decode(lstm_feats)
        # 返回句子分数值, 和最佳解析标签路径

        return score, tag_seq

# 函数sentence_map完成中文文本信息的数字编码, 变成张量

def sentence_map(sentence_list):
    # 初始化数字映射字典

    word_to_ix = {}
    for sentence, tags in sentence_list:
        for word in sentence:
            if word not in word_to_ix:
                word_to_ix[word] = len(word_to_ix)
    return word_to_ix

# 首先设置若干超参数

START_TAG = "<START>"
STOP_TAG = "<STOP>"
EMBEDDING_DIM = 200
HIDDEN_DIM = 100
VOCAB_SIZE = 30

# 初始化标签字典

tag_to_ix = {"O":0, "B-dis":1, "I-dis":2, "B-sym":3, "I-sym":4, START_TAG:5, STOP_TAG:6}

# 输入样本数据

sentence_list = [
    (["女", "性", "，", "8", "8", "岁", "，", "农", "民", "，", "双", "滦", "区", "应", "营", "子", "村", "人", "，", "主", "因", "右", "髋", "部", "摔", "伤", "后", "疼", "痛", "肿", "胀", "，", "活", "动", "受", "限", "5", "小", "时", "于", "2", "0", "1", "6", "-", "1", "0", "-", "2", "9", "；", "1", "1", "：", "1", "2", "入", "院", "。"], ["O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "B-sym", "I-sym", "B-sym", "I-sym", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"]
),
    (["处", "外", "伤", "后", "疼", "痛", "伴", "有", "头", "晕", "2", "小", "时", "于", "2", "0", "1", "6", "-", "-", "1", "0", "-", "-", "0", "2", "1", "2", ":", "0", "2", "收", "入", "院", "。"], ["O", "O", "O", "O", "B-sym", "I-sym", "O", "O", "B-sym", "I-sym", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"]
)]
if __name__ == '__main__':
    # 获取所有字符的映射字典

    word_to_ix = sentence_map(sentence_list)
    # 创建类对象

    model = BiLSTM_CRF(vocab_size=len(word_to_ix), tag_to_ix=tag_to_ix,
                       embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM)

    # 遍历样本数据

    for sentence, tags in sentence_list:
        # 完成自然语言文本到数字化张量的映射

        sentence_in = prepare_sequence(sentence, word_to_ix)
        # 调用类内函数完成发射矩阵张量的计算

        # feat_out = model._get_lstm_features(sentence_in)

        feat_out = model._get_bert_features(sentence_in)
        # 将tags标签进行数字化映射, 然后封装成torch.long型的Tensor张量

        targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long)

        # 直接调用损失值计算函数, 计算loss值

        loss = model.neg_log_likelihood(sentence_in, targets)
        print('loss=', loss)

        # 进入预测阶段, 调用模型默认的forward()函数

        with torch.no_grad():
            # 模型默认的函数就是预测阶段的维特比解码函数

            score, tag_seq = model(sentence_in)
            print(score)
            print(tag_seq)
            print('******')
        

# -*- coding: utf-8 -*-

import torch

# 构建函数评估模型的准确率和召回率

def evaluate(sentence_list, gold_tags, predict_tags, id2char, id2tag):
    """
    description: 评价模型指标方法
    :param sentence_list:    句子列表
    :param gold_tags:        标签序列
    :param predict_tags:     预测标签序列
    :param id2char:          文字码表
    :param id2tag:           标签码表
    :return: step_acc,                   当前批次准确率
             step_recall,                当前批次召回率
             f1_score,                   当前批次 f1 值
             acc_entities_length,        当前批次正确识别实体总数
             predict_entities_length,    当前批次识别出的实体总数
             gold_entities_length        当前批次真实的实体总数
    """

    # 真实实体集合以及每个实体的字与标签列表

    gold_entities, gold_entity = [], []
    # 预测实体集合以及每个实体的字与标签列表

    predict_entities, predict_entity = [], []
    # 遍历当前待评估的句子列表

    for line_no, sentence in enumerate(sentence_list):
        # 迭代句子每个字符

        for char_no in range(len(sentence)):
            # 判断:若句子的id值对应的是0(即:<PAD>)则跳过循环

            if sentence[char_no] == 0:
                break
            # 获取当前句子中的每一个文字

            char_text = id2char[sentence[char_no]]
            # 获取当前字真实实体标注类型

            gold_tag_type = id2tag[gold_tags[line_no][char_no]]
            # 获取当前预测实体标注类型

            predict_tag_type = id2tag[predict_tags[line_no][char_no]]
            # 判断 id2tag 第一个字符是否为 B ,表示实体开始

            if gold_tag_type[0] == "B":
                # 将实体文字与类别加入实体列表中

                gold_entity = [char_text + "/" + gold_tag_type]
            # 判断 id2tag 第一个字符是否为 I ,表示实体中部到结尾

            # 判断依据: I 开头; entiry 不为空; 实体类别相同.

            elif gold_tag_type[0] == "I" and len(gold_entity) != 0 \
                    and gold_entity[-1].split("/")[1][1:] == gold_tag_type[1:]:
                # 满足条件则加入实体列表中

                gold_entity.append(char_text + "/" + gold_tag_type)
            # 判断依据: O 开头, 并且entiry 不为空.

            elif gold_tag_type[0] == "O" and len(gold_entity) != 0 :
                # 增加唯一标识(人为增加的标志位)

                gold_entity.append(str(line_no) + "_" + str(char_no))
                # 将实体文字与类别加入实体列表中

                gold_entities.append(gold_entity)
                # 重置实体列表

                gold_entity=[]
            else:
                # 重置实体列表
                
                gold_entity=[]

            # 判断 id2tag 第一个字符是否为 B ,表示实体开始

            if predict_tag_type[0] == "B":
                # 将实体文字与类别加入实体列表中

                predict_entity = [char_text + "/" + predict_tag_type]
            # 判断 id2tag 第一个字符是否为 I ,表示实体中部到结尾

            # 判断依据: I 开头; entiry 不为空; 实体类别相同.

            elif predict_tag_type[0] == "I" and len(predict_entity) != 0 \
                    and predict_entity[-1].split("/")[1][1:] == predict_tag_type[1:]:
                # 将实体文字与类别加入实体列表中

                predict_entity.append(char_text + "/" + predict_tag_type)
            # 判断依据: O 开头, 并且entiry 不为空.

            elif predict_tag_type[0] == "O" and len(predict_entity) != 0:
                # 增加唯一标识(人为添加的标志位)

                predict_entity.append(str(line_no) + "_" + str(char_no))
                # 将实体加入列表中

                predict_entities.append(predict_entity)
                # 重置实体列表

                predict_entity = []
            else:
                # 重置实体列表

                predict_entity = []

    # 获取预测正确的实体集合

    acc_entities = [entity for entity in predict_entities if entity in gold_entities]
    # 预测正确实体长度(用于计算准确率, 召回, F1值)

    acc_entities_length = len(acc_entities)
    # 预测出实体个数

    predict_entities_length = len(predict_entities)
    # 真实实体列表个数

    gold_entities_length = len(gold_entities)

    # 如果准确实体个数大于0, 则计算准确度, 召回率, f1值

    if acc_entities_length > 0:
        # 准确率

        step_acc = float(acc_entities_length / predict_entities_length)
        # 召回率

        step_recall = float(acc_entities_length / gold_entities_length)
        # f1值

        f1_score = 2 * step_acc * step_recall / (step_acc + step_recall)
        # 返回评估值与各个实体长度(用于整体计算)

        return step_acc, step_recall, f1_score, acc_entities_length, predict_entities_length, gold_entities_length
    else:
        # 准确率, 召回率, f1值均为0

        return 0, 0, 0, acc_entities_length, predict_entities_length, gold_entities_length


# 真实标签数据

tag_list = [
    [0, 0, 3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0],
    [0, 0, 3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0],
    [0, 0, 3, 4, 0, 3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [3, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 0],
    [0, 0, 1, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 3, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
]

# 预测标签数据

predict_tag_list = [
    [0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0],
    [0, 0, 3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0],
    [0, 0, 3, 4, 0, 3, 4, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [3, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 0],
    [0, 0, 1, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0],
    [3, 4, 0, 3, 4, 0, 0, 1, 2, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
    [0, 0, 3, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
]

# 编码与字符对照字典

id2char = {0: '<PAD>', 1: '确', 2: '诊', 3: '弥', 4: '漫', 5: '大', 6: 'b', 7: '细', 8: '胞', 9: '淋', 10: '巴', 11: '瘤',
           12: '1', 13: '年', 14: '反', 15: '复', 16: '咳', 17: '嗽', 18: '、', 19: '痰', 20: '4', 21: '0', 22: ',',
           23: '再', 24: '发', 25: '伴', 26: '气', 27: '促', 28: '5', 29: '天', 30: '。', 31: '生', 32: '长', 33: '育',
           34: '迟', 35: '缓', 36: '9', 37: '右', 38: '侧', 39: '小', 40: '肺', 41: '癌', 42: '第', 43: '三', 44: '次',
           45: '化', 46: '疗', 47: '入', 48: '院', 49: '心', 50: '悸', 51: '加', 52: '重', 53: '胸', 54: '痛', 55: '3',
           56: '闷', 57: '2', 58: '多', 59: '月', 60: '余', 61: ' ', 62: '周', 63: '上', 64: '肢', 65: '无', 66: '力',
           67: '肌', 68: '肉', 69: '萎', 70: '缩', 71: '半'}

# 编码与标签对照字典

id2tag = {0: 'O', 1: 'B-dis', 2: 'I-dis', 3: 'B-sym', 4: 'I-sym'}

# 输入的数字化sentences_sequence, 由下面的sentence_list经过映射函数sentence_map()转化后得到

sentence_list = [
    "确诊弥漫大b细胞淋巴瘤1年",
    "反复咳嗽、咳痰40年,再发伴气促5天。",
    "生长发育迟缓9年。",
    "右侧小细胞肺癌第三次化疗入院",
    "反复气促、心悸10年,加重伴胸痛3天。",
    "反复胸闷、心悸、气促2多月,加重3天",
    "咳嗽、胸闷1月余, 加重1周",
    "右上肢无力3年, 加重伴肌肉萎缩半年"
]

def sentence_map(sentence_list, char_to_id, max_length):
    sentence_list.sort(key=lambda c:len(c), reverse=True)
    sentence_map_list = []
    for sentence in sentence_list:
        sentence_id_list = [char_to_id[c] for c in sentence]
        padding_list = [0] * (max_length-len(sentence))
        sentence_id_list.extend(padding_list)
        sentence_map_list.append(sentence_id_list)
    return torch.tensor(sentence_map_list, dtype=torch.long)

char_to_id = {"<PAD>":0}

SENTENCE_LENGTH = 20

for sentence in sentence_list:
    for _char in sentence:
        if _char not in char_to_id:
            char_to_id[_char] = len(char_to_id)

sentences_sequence = sentence_map(sentence_list, char_to_id, SENTENCE_LENGTH)


if __name__ == '__main__':
    accuracy, recall, f1_score, acc_entities_length, predict_entities_length, true_entities_length = evaluate(sentences_sequence.tolist(), tag_list, predict_tag_list, id2char, id2tag)

    print("accuracy:",                  accuracy,
          "\nrecall:",                  recall,
          "\nf1_score:",                f1_score,
          "\nacc_entities_length:",     acc_entities_length,
          "\npredict_entities_length:", predict_entities_length,
          "\ntrue_entities_length:",    true_entities_length)

# 导入工具包
import os
import torch
import json
from tqdm import tqdm
from bilstm_crf import BiLSTM_CRF

# 完成中文文本数字化映射的函数

def prepare_sequence(seq, char_to_id):
    char_ids = []
    for idx, char in enumerate(seq):
        # 判断若字符不在码表对应字典中, 则取NUK的编码(即unknown), 否则取对应的字符编码

        if char_to_id.get(char):
            char_ids.append(char_to_id[char])
        else:
            char_ids.append(char_to_id["UNK"])
    return torch.tensor(char_ids, dtype=torch.long)

# 单样本预测的函数

def singel_predict(model_path,
                   content,
                   char_to_id_json_path,
                   embedding_dim,
                   hidden_dim,
                   target_type_list,
                   tag_to_id):
    """
    函数功能:                单句命名实体识别预测，返回实体列表
    model_path:             模型文件路径
    content:                待预测文本
    char_to_id_json_path:   字符码表文件路径
    embedding_dim:          字向量维度
    hidden_dim:             BiLSTM 隐藏层向量维度
    target_type_list:       待匹配类型，符合条件的实体将会被提取出来
    tag_to_id:              标签码表对照字典，标签对应 id
    """

    # 加载数字化映射字典

    char_to_id = json.load(open(char_to_id_json_path, mode="r", encoding="utf8"))
    # 实例化类对象

    model = BiLSTM_CRF(len(char_to_id), tag_to_id, embedding_dim, hidden_dim)
    # 加载已经训练好的模型

    model.load_state_dict(torch.load(model_path))
    # 获取需要提取的 tag 对应的 id 列表

    tag_id_dict = {v: k for k, v in tag_to_id.items() if k[2:] in target_type_list}
    # 定义返回实体列表

    entities = []
    # 预测阶段不求解梯度, 不更新参数

    with torch.no_grad():
        # 将待预测的文本进行数字化映射

        sentence_in = prepare_sequence(content, char_to_id)
        # 直接调用模型进行维特比解码

        score, best_path_list = model(sentence_in)
        entity = None
        for char_idx, tag_id in enumerate(best_path_list):
            # 若预测结果 tag_id 属于目标字典数据 key 中

            if tag_id in tag_id_dict:
                # 取符合匹配字典id的第一个字符，即[B, I]

                tag_index = tag_id_dict[tag_id][0]
                # 计算当前字符确切的下标位置

                current_char = content[char_idx]
                # 若当前字标签起始为B, 则设置为实体开始

                if tag_index == "B":
                    entity = current_char
                # 若当前字标签起始为I, 则进行字符串追加

                elif tag_index == "I" and entity:
                    entity += current_char
            # 当实体不为空且当前标签类型为O时, 加入实体列表

            if tag_id == tag_to_id["O"] and entity:
                # 满足当前字符为O, 上一个字符为目标提取实体结尾时, 将其加入实体列表

                if "、" not in entity and "～" not in entity and "。" not in entity \
                    and "”" not in entity and "：" not in entity and ":" not in entity \
                    and "，" not in entity and "," not in entity and "." not in entity \
                    and ";" not in entity and "；" not in entity and "【" not in entity \
                    and "】" not in entity and "[" not in entity and "]" not in entity:
                    entities.append(entity)
                # 重置为空, 准备下一个实体的识别

                entity = None
    # 返回单条文本中所有识别出来的命名实体, 以集合的形式返回

    return set(entities)

def batch_predict(data_path,
                  model_path,
                  char_to_id_json_path,
                  embedding_dim,
                  hidden_dim,
                  target_type_list,
                  prediction_result_path,
                  tag_to_id):
    """
    函数功能: 批量预测, 查询文件目录下数据,
              从中提取符合条件的实体并存储至新的目录[prediction_result_path]
    data_path:               数据文件路径
    model_path:              模型文件路径
    char_to_id_json_path:    字符码表文件路径
    embedding_dim:           字向量维度
    hidden_dim:              隐藏层维度
    target_type_list:        待匹配标签类型
    prediction_result_path:  预测结果保存路径
    tag_to_id:               标签映射字典
    """

    # 迭代路径, 读取文件名

    for fn in tqdm(os.listdir(data_path)):
        # 拼装全路径

        fullpath = os.path.join(data_path, fn)
        # 定义输出结果文件

        entities_file = open(os.path.join(prediction_result_path),
                             mode="w",
                             encoding="utf-8")
        with open(fullpath, mode="r", encoding="utf-8") as f:
            # 读取文件内容

            content = f.readline()
            # 调用单个预测模型，输出为目标类型实体文本列表

            entities = singel_predict(model_path,
                                      content,
                                      char_to_id_json_path,
                                      embedding_dim,
                                      hidden_dim,
                                      target_type_list,
                                      tag_to_id)
            # 写入识别结果文件

            entities_file.write("\n".join(entities))
    print("batch_predict Finished".center(100, "-"))

if __name__ == '__main__':
    # 模型保存路径

    model_path = "model/bilstm_crf_state_dict_20210518_092028.pt"

    # 字向量维度

    EMBEDDING_DIM = 200

    # 隐层维度

    HIDDEN_DIM = 100

    # 标签码表对照字典

    tag_to_id = {"O": 0, "B-dis": 1, "I-dis": 2, "B-sym": 3, "I-sym": 4, "<START>": 5, "<STOP>": 6}

    # 字符码表文件路径

    char_to_id_json_path = "data/char_to_id.json"

    # 预测结果存储路径

    prediction_result_path = "prediction_result"

    # 待匹配标签类型

    target_type_list = ["sym"]

    # 待预测文本文件所在目录

    data_path = "/home/test9504/perl5/dd/ai-doctor/doctor_offline/doctor_data/unstructured/norecognite"

    # 批量文本预测, 并将结果写入文件中

    batch_predict(data_path,
                  model_path,
                  char_to_id_json_path,
                  EMBEDDING_DIM,
                  HIDDEN_DIM,
                  target_type_list,
                  prediction_result_path,
                  tag_to_id)

# -*- coding: utf-8 -*-

# 导入代码所需的全部包

import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.optim as optim
# 导入自定义的模型和评估函数

from bilstm_crf import BiLSTM_CRF
import evaluate
# 导入若干工具包

from tqdm import tqdm
import json
import random
import time
import matplotlib.pyplot as plt

device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu")

# 构建文本到id的映射函数

def prepare_sequence(seq, char_to_id):
    # 初始化空列表

    char_ids = []
    for idx, char in enumerate(seq):
        # 判断若字符不在码表对应字典中，则取 NUK 的编码（即 unknown），否则取对应的字符编>码

        if char_to_id.get(char):
            char_ids.append(char_to_id[char])
        else:
            char_ids.append(char_to_id["UNK"])
    # 将列表封装成Tensor类型返回

    return torch.tensor(char_ids, dtype=torch.long)

# 获取训练数据集和验证数据集

def get_data():
    # 训练数据集和验证数据集的路径, 后面直接以读取文件的形式获取数据即可

    train_data_file_path = "data/train_data.txt"
    validate_file_path = "data/validate_data.txt"
    train_data_list = []
    validate_data_list = []
    # 因为每一行都是一个样本, 所以按行遍历即可

    for line in open(train_data_file_path, mode="r", encoding="utf8"):
        # 每行样本数据都是json字符串, 直接loads进来, 再追加进结果列表中

        data = json.loads(line)
        train_data_list.append(data)
    for line in open(validate_file_path, mode="r", encoding="utf8"):
        # 采用和上面训练数据同样的处理方法即可

        data = json.loads(line)
        validate_data_list.append(data)
    # 以列表形式返回训练数据集和验证数据集

    return train_data_list, validate_data_list

def save_train_history_image(train_history_list,
                             validate_history_list,
                             history_image_path,
                             data_type):
    """
    description: 存储训练历史图片
    :param train_history_list:          训练历史结果
    :param validate_history_list:       验证历史结果
    :param history_image_path:          历史数据生成图像保存路径
    :param data_type:                   数据类型[用于替换label,y轴以及保存文件名中数据类型
]
    :return:                            无,直接将数据转为图片进行存储
    """

    # 存储训练历史图片

    plt.plot(train_history_list, label="Train %s History" % (data_type))
    plt.plot(validate_history_list, label="Validate %s History" % (data_type))
    plt.legend(loc="best")
    plt.xlabel("Epochs")
    plt.ylabel(data_type)
    plt.savefig(history_image_path.replace("plot", data_type))
    plt.close()





# 训练主函数代码

if __name__ == '__main__':
    # 先将超参数和训练数据导入, 所有的预备工作准备好

    # EMBEDDING_DIM = 200

    EMBEDDING_DIM = 768
    HIDDEN_DIM = 100
    train_data_list, validate_data_list = get_data()
    # 字符到id的映射提前通过全样本统计完了, 也可以放在单独的函数中构建char_to_id

    char_to_id = json.load(open('./data/char_to_id.json', mode='r', encoding='utf-8'))
    # 标签到id的映射是由特定任务决定的, 任务一旦确定那么这个字典也就确定了, 因此也以超参数的形式提前准备好

    tag_to_ix = {"O": 0, "B-dis": 1, "I-dis": 2, "B-sym": 3, "I-sym": 4, "<START>": 5, "<STOP>": 6}

    # 直接构建模型对象

    # MODEL_PATH = './model/bilstm_crf_state_dict_20210519_210902.pt'

    model = BiLSTM_CRF(len(char_to_id), tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM).to(device)
    # model.load_state_dict(torch.load(MODEL_PATH))

    # 直接选定优化器

    # optimizer = optim.SGD(model.parameters(), lr=0.5, momentum=0.85, weight_decay=1e-4)

    optimizer = optim.Adam(model.parameters(), lr=0.05)

    # 调转字符标签与id值

    id_to_tag = {v: k for k, v in tag_to_ix.items()}
    # 调转字符编码与id值

    id_to_char = {v: k for k, v in char_to_id.items()}

    # 获取时间戳标签, 用于模型文件和图片, 日志文件的命名

    time_str = time.strftime("%Y%m%d_%H%M%S", time.localtime(time.time()))
    model_saved_path = "model/bilstm_crf_state_dict_%s.pt" % (time_str)
    train_history_image_path = "log/bilstm_crf_train_plot_%s.png" % (time_str)
    log_file = open("log/train_%s.log"%(time_str), mode="w", encoding="utf8")
    # 设定整个训练数据的批次数

    epochs = 20
    # 将几个重要的统计量做初始化

    train_loss_history, train_acc_history, train_recall_history, train_f1_history = [], [], [], []
    validate_loss_history, validate_acc_history, validate_recall_history, validate_f1_history = [], [], [], []

    # 按epoch进行迭代训练

    for epoch in range(epochs):
        tqdm.write("Epoch {}/{}".format(epoch + 1, epochs))
        total_acc_length, total_prediction_length, total_gold_length, total_loss = 0, 0, 0, 0
        # 每一个epoch先遍历训练集, 通过训练样本更新模型

        for train_data in tqdm(train_data_list):
            model.zero_grad()
            # 取出原始样本数据

            sentence, tags = train_data.get("text"), train_data.get("label")
            # 完成数字化编码的映射

            # sentence_in = prepare_sequence(sentence, char_to_id)

            # print(sentence_in)

            sentence_in = model.bert.get_token_for_single(sentence)
            # print(sentence_in)

            targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long).to(device)
            # 直接将特征和标签送入模型中计算loss

            # 注意: model中真实计算损失值的函数不是forward(), 而是neg_log_likelihood()

            loss = model.neg_log_likelihood(sentence_in, targets)
            loss.backward()
            optimizer.step()
            # 这一步才真正的调用forward()函数, 目的是通过维特比算法解码出最佳路径, 本质上是一个预测的过程

            score, best_path_list = model(sentence_in.to(device))
            # 累加每一个样本的损失值

            step_loss = loss.data.cpu().numpy()
            total_loss += step_loss
            sentence_in_unq = sentence_in.unsqueeze(0)
            targets_unq = targets.unsqueeze(0)
            best_path_list_up = [best_path_list]
            step_acc, step_recall,  f1_score,  acc_entities_length,  predict_entities_length, gold_entities_length = evaluate.evaluate(sentence_in_unq.tolist(), targets_unq.tolist(), best_path_list_up, id_to_char, id_to_tag)
            # 累加三个重要的统计量, 预测正确的实体数, 预测的总实体数, 真实标签的实体数

            total_acc_length += acc_entities_length
            total_prediction_length += predict_entities_length
            total_gold_length += gold_entities_length

        print("train:", total_acc_length, total_prediction_length, total_gold_length)
        # 如果for循环结束, 真实预测的实体数大于0, 则计算准确率, 召回率, F1等统计量值

        if total_prediction_length > 0:
            train_mean_loss = total_loss / len(train_data_list)
            train_epoch_acc = total_acc_length / total_prediction_length
            train_epoch_recall = total_acc_length / total_gold_length
            train_epoch_f1 = 2 * train_epoch_acc * train_epoch_recall / (train_epoch_acc + train_epoch_recall)
        else:
            log_file.write("train_total_prediction_length is zero" + "\n")

        # 每一个epoch, 训练结束后直接在验证集上进行验证

        total_acc_length, total_prediction_length, total_gold_length, total_loss = 0, 0, 0, 0
        # 进入验证阶段最重要的一步就是保持模型参数不变, 不参与反向传播和参数更新

        with torch.no_grad():
            for validate_data in tqdm(validate_data_list):
                # 直接提取验证集的特征和标签, 并进行数字化映射

                sentence, tags = validate_data.get("text"), validate_data.get("label")
                # sentence_in = prepare_sequence(sentence, char_to_id)

                sentence_in = model.bert.get_token_for_single(sentence)
                targets = torch.tensor([tag_to_ix[t] for t in tags], dtype=torch.long).to(device)
                # 验证阶段的损失值依然是通过neg_log_likelihood函数进行计算

                loss = model.neg_log_likelihood(sentence_in, targets)
                # 验证阶段的解码依然是通过直接调用model中的forward函数进行

                score, best_path_list = model(sentence_in.to(device))
                # 累加每一个样本的损失值

                step_loss = loss.data.cpu().numpy()
                total_loss += step_loss
                sentence_in_unq = sentence_in.unsqueeze(0)
                targets_unq = targets.unsqueeze(0)
                best_path_list_up = [best_path_list]
                step_acc, step_recall,  f1_score,  acc_entities_length,  predict_entities_length, gold_entities_length = evaluate.evaluate(sentence_in_unq.tolist(), targets_unq.tolist(), best_path_list_up, id_to_char, id_to_tag)
                # 累加三个重要的统计量, 预测正确的实体数, 预测的总实体数, 真实标签的实体数

                total_acc_length += acc_entities_length
                total_prediction_length += predict_entities_length
                total_gold_length += gold_entities_length

        print("validate:", total_acc_length, total_prediction_length, total_gold_length)
        # 当准确预测的数量大于0, 并且总的预测标签量大于0, 计算验证集上的准确率, 召回率, F1值

        if total_acc_length != 0 and total_prediction_length != 0:
            validate_mean_loss = total_loss / len(validate_data_list)
            validate_epoch_acc = total_acc_length / total_prediction_length
            validate_epoch_recall = total_acc_length / total_gold_length
            validate_epoch_f1 = 2 * validate_epoch_acc * validate_epoch_recall / (validate_epoch_acc + validate_epoch_recall)
            log_text = "Epoch: %s | train loss: %.5f |train acc: %.3f |train recall: %.3f |train f1 score: %.3f" \
                       " | validate loss: %.5f |validate acc: %.3f |validate recall: %.3f |validate f1 score: %.3f" % \
                       (epoch,
                        train_mean_loss, train_epoch_acc, train_epoch_recall, train_epoch_f1,
                        validate_mean_loss, validate_epoch_acc, validate_epoch_recall, validate_epoch_f1)
            log_file.write(log_text+"\n")

            # 将当前轮次的重要统计量添加进列表中, 为后续的画图做准备

            train_loss_history.append(train_mean_loss)
            train_acc_history.append(train_epoch_acc)
            train_recall_history.append(train_epoch_recall)
            train_f1_history.append(train_epoch_f1)
            validate_loss_history.append(validate_mean_loss)
            validate_acc_history.append(validate_epoch_acc)
            validate_recall_history.append(validate_epoch_recall)
            validate_f1_history.append(validate_epoch_f1)
        else:
            log_file.write("validate_total_prediction_length is zero" + "\n")

    # 保存模型

    torch.save(model.state_dict(), model_saved_path)

    # 将 loss 下降历史数据转为图片存储

    save_train_history_image(train_loss_history,
                             validate_loss_history,
                             train_history_image_path,
                             "Loss")
    # 将准确率提升历史数据转为图片存储

    save_train_history_image(train_acc_history,
                             validate_acc_history,
                             train_history_image_path,
                             "Acc")
    # 将召回提升历史数据转为图片存储

    save_train_history_image(train_recall_history,
                             validate_recall_history,
                             train_history_image_path,
                             "Recall")
    # 将F1上升历史数据转为图片存储

    save_train_history_image(train_f1_history,
                             validate_f1_history,
                             train_history_image_path,
                             "F1")
    print("train Finished".center(100, "-"))

    # 训练代码到此结束了, 最后打印一下训练好的模型中各个组成模块的参数详情, 以便开发人员核对

    for name, parameters in model.named_parameters():
        print(name, ':', parameters.size())

# if __name__ == '__main__':
#     train_data_list, valid_data_list = get_data()
#     print(train_data_list[:5])
#     print(len(train_data_list))
#     print('******')
#     print(valid_data_list[:5])
#     print(len(valid_data_list))