''' Script for downloading all GLUE data.'''

import os
import sys
import shutil
import argparse
import tempfile
import urllib.request
import zipfile

TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"]
TASK2PATH = {"CoLA":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4',
             "SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
             "MRPC":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc',
             "QQP":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5',
             "STS":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5',
             "MNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce',
             "SNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df',
             "QNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601',
             "RTE":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb',
             "WNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf',
             "diagnostic":'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'}

MRPC_TRAIN = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt'
MRPC_TEST = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt'

def download_and_extract(task, data_dir):
    print("Downloading and extracting %s..." % task)
    data_file = "%s.zip" % task
    urllib.request.urlretrieve(TASK2PATH[task], data_file)
    with zipfile.ZipFile(data_file) as zip_ref:
        zip_ref.extractall(data_dir)
    os.remove(data_file)
    print("\tCompleted!")

def format_mrpc(data_dir, path_to_data):
    print("Processing MRPC...")
    mrpc_dir = os.path.join(data_dir, "MRPC")
    if not os.path.isdir(mrpc_dir):
        os.mkdir(mrpc_dir)
    if path_to_data:
        mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt")
        mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt")
    else:
        print("Local MRPC data not specified, downloading data from %s" % MRPC_TRAIN)
        mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt")
        mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt")
        urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file)
        urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file)
    assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file
    assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file
    urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv"))

    dev_ids = []
    with open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding="utf8") as ids_fh:
        for row in ids_fh:
            dev_ids.append(row.strip().split('\t'))

    with open(mrpc_train_file, encoding="utf8") as data_fh, \
         open(os.path.join(mrpc_dir, "train.tsv"), 'w', encoding="utf8") as train_fh, \
         open(os.path.join(mrpc_dir, "dev.tsv"), 'w', encoding="utf8") as dev_fh:
        header = data_fh.readline()
        train_fh.write(header)
        dev_fh.write(header)
        for row in data_fh:
            label, id1, id2, s1, s2 = row.strip().split('\t')
            if [id1, id2] in dev_ids:
                dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2))
            else:
                train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2))

    with open(mrpc_test_file, encoding="utf8") as data_fh, \
            open(os.path.join(mrpc_dir, "test.tsv"), 'w', encoding="utf8") as test_fh:
        header = data_fh.readline()
        test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n")
        for idx, row in enumerate(data_fh):
            label, id1, id2, s1, s2 = row.strip().split('\t')
            test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2))
    print("\tCompleted!")

def download_diagnostic(data_dir):
    print("Downloading and extracting diagnostic...")
    if not os.path.isdir(os.path.join(data_dir, "diagnostic")):
        os.mkdir(os.path.join(data_dir, "diagnostic"))
    data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv")
    urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file)
    print("\tCompleted!")
    return

def get_tasks(task_names):
    task_names = task_names.split(',')
    if "all" in task_names:
        tasks = TASKS
    else:
        tasks = []
        for task_name in task_names:
            assert task_name in TASKS, "Task %s not found!" % task_name
            tasks.append(task_name)
    return tasks

def main(arguments):
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data')
    parser.add_argument('--tasks', help='tasks to download data for as a comma separated string',
                        type=str, default='all')
    parser.add_argument('--path_to_mrpc', help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt',
                        type=str, default='')
    args = parser.parse_args(arguments)

    if not os.path.isdir(args.data_dir):
        os.mkdir(args.data_dir)
    tasks = get_tasks(args.tasks)

    for task in tasks:
        if task == 'MRPC':
            format_mrpc(args.data_dir, args.path_to_mrpc)
        elif task == 'diagnostic':
            download_diagnostic(args.data_dir)
        else:
            download_and_extract(task, args.data_dir)


if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))

# 假设你已经将以上代码copy到download_glue_data.py文件中

# 运行这个python脚本, 你将同目录下得到一个glue文件夹

python download_glue_data.py

import torch

# 预训练模型来源

source = 'huggingface/pytorch-transformers'
# 选定加载模型的哪一部分, 这里是模型的映射器

part = 'tokenizer'
# 加载的预训练模型的名字

model_name = 'bert-base-chinese'
tokenizer = torch.hub.load(source, part, model_name)  

# 加载不带头的预训练模型

part = 'model'
model = torch.hub.load(source, part, model_name)

# 加载带有语言模型头的预训练模型

part = 'modelWithLMHead'
lm_model = torch.hub.load(source, part, model_name)

# 加载带有类模型头的预训练模型

part = 'modelForSequenceClassification'
classification_model = torch.hub.load(source, part, model_name)

# 加载带有问答模型头的预训练模型

part = 'modelForQuestionAnswering'
qa_model = torch.hub.load(source, part, model_name)

# 输入的中文文本

input_text = "人生该如何起头"

# 使用tokenizer进行数值映射

indexed_tokens = tokenizer.encode(input_text)

# 打印映射后的结构

print("indexed_tokens:", indexed_tokens)

# 将映射结构转化为张量输送给不带头的预训练模型

tokens_tensor = torch.tensor([indexed_tokens])

# 使用不带头的预训练模型获得结果

with torch.no_grad():
    encoded_layers, _ = model(tokens_tensor)

print("不带头的模型输出结果:", encoded_layers)

print("不带头的模型输出结果的尺寸:", encoded_layers.shape)

# 使用带有语言模型头的预训练模型获得结果

with torch.no_grad():
    lm_output = lm_model(tokens_tensor)

print("带语言模型头的模型输出结果:", lm_output)

print("带语言模型头的模型输出结果的尺寸:", lm_output[0].shape)

# 使用带有分类模型头的预训练模型获得结果

with torch.no_grad():
    classification_output = classification_model(tokens_tensor)

print("带分类模型头的模型输出结果:", classification_output)

print("带分类模型头的模型输出结果的尺寸:", classification_output[0].shape)

# 使用带有问答模型头的模型进行输出时, 需要使输入的形式为句子对
# 第一条句子是对客观事物的陈述
# 第二条句子是针对第一条句子提出的问题
# 问答模型最终将得到两个张量, 
# 每个张量中最大值对应索引的分别代表答案的在文本中的起始位置和终止位置.

input_text1 = "我家的小狗是黑色的"
input_text2 = "我家的小狗是什么颜色的呢?"


# 映射两个句子
indexed_tokens = tokenizer.encode(input_text1, input_text2)
print("句子对的indexed_tokens:", indexed_tokens)

# 输出结果: [101, 2769, 2157, 4638, 2207, 4318, 3221, 7946, 5682, 4638, 102, 2769, 2157, 4638, 2207, 4318, 3221, 784, 720, 7582, 5682, 4638, 1450, 136, 102]

# 用0，1来区分第一条和第二条句子

segments_ids = [0]*11 + [1]*14

# 转化张量形式

segments_tensors = torch.tensor([segments_ids])
tokens_tensor = torch.tensor([indexed_tokens])

# 使用带有问答模型头的预训练模型获得结果

with torch.no_grad():
    start_logits, end_logits = qa_model(tokens_tensor, token_type_ids=segments_tensors)


print("带问答模型头的模型输出结果:", (start_logits, end_logits))
print("带问答模型头的模型输出结果的尺寸:", (start_logits.shape, end_logits.shape))

# 克隆huggingface的transfomers文件

git clone https://github.com/huggingface/transformers.git

# 进行transformers文件夹

cd transformers

# 安装python的transformer工具包, 因为微调脚本是py文件.

pip install .

# 当前的版本可能跟我们教学的版本并不相同，你还需要执行：


pip install transformers==2.3.0

# 进入微调脚本所在路径并查看

cd examples
ls

# 其中run_glue.py就是针对GLUE数据集合任务类型的微调脚本

# 定义DATA_DIR: 微调数据所在路径, 这里我们使用glue_data中的数据作为微调数据

export DATA_DIR="../../glue_data"
# 定义SAVE_DIR: 模型的保存路径, 我们将模型保存在当前目录的bert_finetuning_test文件中

export SAVE_DIR="./bert_finetuning_test/"

# 使用python运行微调脚本
# --model_type: 选择需要微调的模型类型, 这里可以选择BERT, XLNET, XLM, roBERTa, distilBERT, ALBERT
# --model_name_or_path: 选择具体的模型或者变体, 这里是在英文语料上微调, 因此选择bert-base-uncased
# --task_name: 它将代表对应的任务类型, 如MRPC代表句子对二分类任务
# --do_train: 使用微调脚本进行训练
# --do_eval: 使用微调脚本进行验证
# --data_dir: 训练集及其验证集所在路径, 将自动寻找该路径下的train.tsv和dev.tsv作为训练集和验证集
# --max_seq_length: 输入句子的最大长度, 超过则截断, 不足则补齐
# --learning_rate: 学习率
# --num_train_epochs: 训练轮数
# --output_dir $SAVE_DIR: 训练后的模型保存路径
# --overwrite_output_dir: 再次训练时将清空之前的保存路径内容重新写入

python run_glue.py \
  --model_type BERT \
  --model_name_or_path bert-base-uncased \
  --task_name MRPC \
  --do_train \
  --do_eval \
  --data_dir $DATA_DIR/MRPC/ \
  --max_seq_length 128 \
  --learning_rate 2e-5 \
  --num_train_epochs 1.0 \
  --output_dir $SAVE_DIR \
  --overwrite_output_dir

# 默认账户

username: ItcastAI
password: ItcastAI

import torch

source = 'huggingface/pytorch-transformers'
# 模型名字为'ItcastAI/bert_cn_finetuning'

model_name = 'ItcastAI/bert_cn_finetuning'

tokenizer = torch.hub.load(source, 'tokenizer', model_name)
model =  torch.hub.load(source, 'modelForSequenceClassification', model_name)

def get_label(text):
    index = tokenizer.encode(text)
    tokens_tensor = torch.tensor([index])
    # 使用评估模式

    with torch.no_grad():
        # 使用模型预测获得结果

        result = model(tokens_tensor)
    predicted_label = torch.argmax(result[0]).item()
    return predicted_label

if __name__ == "__main__":
    # text = "早餐不好,服务不到位,晚餐无西餐,早餐晚餐相同,房间条件不好"

    text = "房间应该超出30平米,是HK同级酒店中少有的大;重装之后,设备也不错."
    print("输入文本为:", text)
    print("预测标签为:", get_label(text))

import torch
# 进行句子的截断补齐(规范长度)

from keras.preprocessing import sequence

source = 'huggingface/pytorch-transformers'

# 直接使用预训练的bert中文模型

model_name = 'bert-base-chinese'

# 通过torch.hub获得已经训练好的bert-base-chinese模型

model =  torch.hub.load(source, 'model', model_name)

# 获得对应的字符映射器, 它将把中文的每个字映射成一个数字

tokenizer = torch.hub.load(source, 'tokenizer', model_name)

# 句子规范长度

cutlen = 32

def get_bert_encode(text):
    """
    description: 使用bert-chinese编码中文文本
    :param text: 要进行编码的文本
    :return: 使用bert编码后的文本张量表示
    """

    # 首先使用字符映射器对每个汉字进行映射

    # 这里需要注意, bert的tokenizer映射后会为结果前后添加开始和结束标记即101和102 

    # 这对于多段文本的编码是有意义的, 但在我们这里没有意义, 因此使用[1:-1]对头和尾进行切片

    indexed_tokens = tokenizer.encode(text[:cutlen])[1:-1]
    # 对映射后的句子进行截断补齐

    indexed_tokens = sequence.pad_sequences([indexed_tokens], cutlen) 
    # 之后将列表结构转化为tensor

    tokens_tensor = torch.LongTensor(indexed_tokens)
    # 使模型不自动计算梯度

    with torch.no_grad():
        # 调用模型获得隐层输出

        encoded_layers, _ = model(tokens_tensor)
    # 输出的隐层是一个三维张量, 最外层一维是1, 我们使用[0]降去它.

    encoded_layers = encoded_layers[0]
    return encoded_layers

if __name__ == "__main__":
    text = "早餐不好,服务不到位,晚餐无西餐,早餐晚餐相同,房间条件不好"
    encoded_layers = get_bert_encode(text)
    print(encoded_layers)
    print(encoded_layers.shape)

import torch.nn as nn
import torch.nn.functional as F


class Net(nn.Module):
    """定义微调网络的类"""

    def __init__(self, char_size=32, embedding_size=768):
        """
        :param char_size: 输入句子中的字符数量, 即输入句子规范后的长度128.
        :param embedding_size: 字嵌入的维度, 因为使用的bert中文模型嵌入维度是768, 因此embedding_size为768
        """

        super(Net, self).__init__()
        # 将char_size和embedding_size传入其中

        self.char_size = char_size
        self.embedding_size = embedding_size
        # 实例化一个全连接层

        self.fc1 = nn.Linear(char_size*embedding_size, 2)

    def forward(self, x):
        # 对输入的张量形状进行变换, 以满足接下来层的输入要求

        x = x.view(-1, self.char_size*self.embedding_size)
        # 使用一个全连接层

        x = self.fc1(x)
        return x

if __name__ == "__main__":
    # 随机初始化一个输入参数

    x = torch.randn(1, 32, 768)
    # 实例化网络结构, 所有参数使用默认值

    net = Net()
    nr = net(x)
    print(nr)    

import pandas as pd
from collections import Counter
from functools import reduce
from sklearn.utils import shuffle

def data_loader(train_data_path, valid_data_path, batch_size):
    """
    description: 从持久化文件中加载数据
    :param train_data_path: 训练数据路径
    :param valid_data_path: 验证数据路径
    :param batch_size: 训练和验证数据集的批次大小
    :return: 训练数据生成器, 验证数据生成器, 训练数据数量, 验证数据数量
    """

    # 使用pd进行csv数据的读取, 并去除第一行的列名

    train_data = pd.read_csv(train_data_path, header=None, sep="\t").drop([0])
    valid_data = pd.read_csv(valid_data_path, header=None, sep="\t").drop([0])

    # 打印训练集和验证集上的正负样本数量

    print("训练数据集的正负样本数量:")
    print(dict(Counter(train_data[1].values)))
    print("验证数据集的正负样本数量:")
    print(dict(Counter(valid_data[1].values)))

    # 验证数据集中的数据总数至少能够满足一个批次

    if len(valid_data) < batch_size:
        raise("Batch size or split not match!")

    def _loader_generator(data):
        """
        description: 获得训练集/验证集的每个批次数据的生成器
        :param data: 训练数据或验证数据
        :return: 一个批次的训练数据或验证数据的生成器
        """
        # 以每个批次的间隔遍历数据集

        for batch in range(0, len(data), batch_size):
            # 定义batch数据的张量列表

            batch_encoded = []
            batch_labels = []
            # 将一个bitch_size大小的数据转换成列表形式, 并进行逐条遍历

            for item in shuffle(data.values.tolist())[batch: batch+batch_size]:
                # 使用bert中文模型进行编码

                encoded = get_bert_encode(item[0])
                # 将编码后的每条数据装进预先定义好的列表中

                batch_encoded.append(encoded)
                # 同样将对应的该batch的标签装进labels列表中

                batch_labels.append([int(item[1])])
            # 使用reduce高阶函数将列表中的数据转换成模型需要的张量形式

            # encoded的形状是(batch_size*max_len, embedding_size)

            encoded = reduce(lambda x, y: torch.cat((x, y), dim=0), batch_encoded)
            labels = torch.tensor(reduce(lambda x, y: x + y, batch_labels))
            # 以生成器的方式返回数据和标签

            yield (encoded, labels)

    # 对训练集和验证集分别使用_loader_generator函数, 返回对应的生成器

    # 最后还要返回训练集和验证集的样本数量

    return _loader_generator(train_data), _loader_generator(valid_data), len(train_data), len(valid_data)

if __name__ == "__main__":
    train_data_path = "./cn_data/SST-2/train.tsv"
    valid_data_path = "./cn_data/SST-2/dev.tsv"
    batch_size = 16
    train_data_labels, valid_data_labels, \
    train_data_len, valid_data_len = data_loader(train_data_path, valid_data_path, batch_size)
    print(next(train_data_labels))
    print(next(valid_data_labels))
    print("train_data_len:", train_data_len)
    print("valid_data_len:", valid_data_len)

import torch.optim as optim

def train(train_data_labels):
    """
    description: 训练函数, 在这个过程中将更新模型参数, 并收集准确率和损失
    :param train_data_labels: 训练数据和标签的生成器对象
    :return: 整个训练过程的平均损失之和以及正确标签的累加数
    """

    # 定义训练过程的初始损失和准确率累加数

    train_running_loss = 0.0
    train_running_acc = 0.0
    # 循环遍历训练数据和标签生成器, 每个批次更新一次模型参数

    for train_tensor, train_labels in train_data_labels:
        # 初始化该批次的优化器

        optimizer.zero_grad()
        # 使用微调网络获得输出

        train_outputs = net(train_tensor)
        # 得到该批次下的平均损失

        train_loss = criterion(train_outputs, train_labels)
        # 将该批次的平均损失加到train_running_loss中

        train_running_loss += train_loss.item()
        # 损失反向传播

        train_loss.backward()
        # 优化器更新模型参数

        optimizer.step()
        # 将该批次中正确的标签数量进行累加, 以便之后计算准确率

        train_running_acc += (train_outputs.argmax(1) == train_labels).sum().item()
    return train_running_loss, train_running_acc


def valid(valid_data_labels):
    """
    description: 验证函数, 在这个过程中将验证模型的在新数据集上的标签, 收集损失和准确率
    :param valid_data_labels: 验证数据和标签的生成器对象
    :return: 整个验证过程的平均损失之和以及正确标签的累加数
    """

    # 定义训练过程的初始损失和准确率累加数

    valid_running_loss = 0.0
    valid_running_acc = 0.0
    # 循环遍历验证数据和标签生成器

    for valid_tensor, valid_labels in valid_data_labels:
        # 不自动更新梯度

        with torch.no_grad():
            # 使用微调网络获得输出

            valid_outputs = net(valid_tensor)
            # 得到该批次下的平均损失

            valid_loss = criterion(valid_outputs, valid_labels)
            # 将该批次的平均损失加到valid_running_loss中

            valid_running_loss += valid_loss.item()
            # 将该批次中正确的标签数量进行累加, 以便之后计算准确率

            valid_running_acc += (valid_outputs.argmax(1) == valid_labels).sum().item()
    return valid_running_loss,  valid_running_acc

if __name__ == "__main__":
    # 设定数据路径

    train_data_path = "./cn_data/SST-2/train.tsv"
    valid_data_path = "./cn_data/SST-2/dev.tsv"
    # 定义交叉熵损失函数

    criterion = nn.CrossEntropyLoss()
    # 定义SGD优化方法

    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
    # 定义训练轮数

    epochs = 4
    # 定义批次样本数量

    batch_size = 16
    # 进行指定轮次的训练

    for epoch in range(epochs):
        # 打印轮次

        print("Epoch:", epoch + 1)
        # 通过数据加载器获得训练数据和验证数据生成器, 以及对应的样本数量

        train_data_labels, valid_data_labels, train_data_len, \
        valid_data_len = data_loader(train_data_path, valid_data_path, batch_size)
        # 调用训练函数进行训练

        train_running_loss, train_running_acc = train(train_data_labels)
        # 调用验证函数进行验证

        valid_running_loss, valid_running_acc = valid(valid_data_labels)
        # 计算每一轮的平均损失, train_running_loss和valid_running_loss是每个批次的平均损失之和

        # 因此将它们乘以batch_size就得到了该轮的总损失, 除以样本数即该轮次的平均损失

        train_average_loss = train_running_loss * batch_size / train_data_len
        valid_average_loss = valid_running_loss * batch_size / valid_data_len

        # train_running_acc和valid_running_acc是每个批次的正确标签累加和,

        # 因此只需除以对应样本总数即是该轮次的准确率

        train_average_acc = train_running_acc /  train_data_len
        valid_average_acc = valid_running_acc / valid_data_len
        # 打印该轮次下的训练损失和准确率以及验证损失和准确率

        print("Train Loss:", train_average_loss, "|", "Train Acc:", train_average_acc)
        print("Valid Loss:", valid_average_loss, "|", "Valid Acc:", valid_average_acc)

    print('Finished Training')

    # 保存路径

    MODEL_PATH = './BERT_net.pth'
    # 保存模型参数

    torch.save(net.state_dict(), MODEL_PATH) 
    print('Finished Saving')    

if __name__ == "__main__":
    MODEL_PATH = './BERT_net.pth'
    # 加载模型参数

    net.load_state_dict(torch.load(MODEL_PATH))

    # text = "酒店设备一般，套房里卧室的不能上网，要到客厅去。"

    text = "房间应该超出30平米,是HK同级酒店中少有的大;重装之后,设备也不错."
    print("输入文本为:", text)
    with torch.no_grad():
        output = net(get_bert_encode(text))
        # 从output中取出最大值对应的索引
        
        print("预测标签为:", torch.argmax(output).item())