NLP - 快速上手¶

from functools import partial
import numpy as np

import paddle
from paddlenlp.datasets import load_dataset
from paddlenlp.transformers import BertForSequenceClassification, AutoTokenizer
# 定义数据集
train_ds = load_dataset("chnsenticorp", splits=["train"])
# 定义模型
model = BertForSequenceClassification.from_pretrained("bert-wwm-chinese", num_classes=len(train_ds.label_list))
# 定义 Tokenizer，实现文本到 ID 的转化
tokenizer = AutoTokenizer.from_pretrained("bert-wwm-chinese")
# 数据处理，从文本到处理好的数据，input_ids，token_type_ids
def convert_example(example, tokenizer):
    encoded_inputs = tokenizer(text=example["text"], max_seq_len=512, pad_to_max_seq_len=True)
    return tuple([np.array(x, dtype="int64") for x in [
            encoded_inputs["input_ids"], encoded_inputs["token_type_ids"], [example["label"]]]])

train_ds = train_ds.map(partial(convert_example, tokenizer=tokenizer))
# 定义 BatchSampler，组合 batch，shuffle 数据
batch_sampler = paddle.io.BatchSampler(dataset=train_ds, batch_size=8, shuffle=True)
train_data_loader = paddle.io.DataLoader(dataset=train_ds, batch_sampler=batch_sampler, return_list=True)
# 定义 optimizer 优化器
optimizer = paddle.optimizer.AdamW(learning_rate=0.001, parameters=model.parameters())
# 定义 loss
criterion = paddle.nn.loss.CrossEntropyLoss()
# 训练
for input_ids, token_type_ids, labels in train_data_loader():
    logits = model(input_ids, token_type_ids)
    loss = criterion(logits, labels)
    loss.backward()
    optimizer.step()
    optimizer.clear_grad()

import numpy as np
import pandas as pd
import torch
from datasets import Dataset
from transformers import BertTokenizer
from transformers import DataCollatorWithPadding
from transformers import BertForSequenceClassification
from torch.optim import AdamW

from datasets import load_dataset
# 定义模型
model = BertForSequenceClassification.from_pretrained(
        "hfl/chinese-bert-wwm-ext", num_labels=2)
# 定义 Tokenizer，实现文本到 ID 的转化
tokenizer = BertTokenizer.from_pretrained("hfl/chinese-bert-wwm-ext")

# 数据处理，从文本到处理好的数据，input_ids，token_type_ids
def preprocess_function(examples):
    result = tokenizer(
        examples["text"],
        padding=False,
        max_length=128,
        truncation=True,
        return_token_type_ids=True, )
    if "label" in examples:
        result["labels"] = [examples["label"]]
    return result
# 定义数据集
dataset = load_dataset("seamew/ChnSentiCorp")["train"]
dataset = dataset.map(
    preprocess_function,
    batched=False,
    remove_columns=dataset.column_names,
    desc="Running tokenizer on dataset", )
dataset.set_format(
    "np", columns=["input_ids", "token_type_ids", "labels"])

# 定义 BatchSampler，组合 batch，shuffle 数据
sampler = torch.utils.data.SequentialSampler(dataset)
collate_fn = DataCollatorWithPadding(tokenizer)
data_loader = torch.utils.data.DataLoader(
    dataset,
    batch_size=4,
    sampler=sampler,
    num_workers=0,
    collate_fn=collate_fn, )

# 定义 loss
criterion = torch.nn.CrossEntropyLoss()
# 定义 optimizer 优化器
optimizer = AdamW(params=model.parameters(), lr=3e-5)

# 训练
for batch in data_loader:
    output = model(**batch).logits
    loss = criterion(output, batch['labels'].reshape(-1))
    loss.backward()
    print(loss)
    optimizer.step()
    optimizer.zero_grad()

def _init_weights(self, module):
        """初始化权重"""
        #分情况讨论模型具体层
        if isinstance(module, nn.Linear):
            # cf https://github.com/pytorch/pytorch/pull/5617
            module.weight.data.normal_(mean=0.0,
                                       std=self.config.initializer_range)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0,
                                       std=self.config.initializer_range)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)

def _init_weights(self, module):
        """初始化权重"""
        normal_init = nn.initializer.Normal(mean=0.0,
                                            std=self.config.initializer_range)
        zero_init = nn.initializer.Constant(0.)
        one_init = nn.initializer.Constant(1.)
        #分情况讨论模型具体层
        if isinstance(module, nn.Linear):
            normal_init(module.weight)
            if module.bias is not None:
                zero_init(module.bias)
        elif isinstance(module, nn.Embedding):
            normal_init(module.weight)
            if module._padding_idx is not None:
                with paddle.no_grad():
                    module.weight[module._padding_idx] = 0
        elif isinstance(module, nn.LayerNorm):
            zero_init(module.bias)
            one_init(module.weight)

from transformers import BertModel
import torch

hf_model = BertModel.from_pretrained("bert-base-uncased")
hf_model.eval()
PATH = './torch_weight.bin'
torch.save(hf_model.state_dict(), PATH)

def convert_pytorch_checkpoint_to_paddle(
    pytorch_checkpoint_path="pytorch_model.bin",
    paddle_dump_path="model_state.pdparams",
    version="old",
):
    do_not_transpose = []
    if version == "old":
        hf_to_paddle.update({
            "predictions.bias": "predictions.decoder_bias",
            ".gamma": ".weight",
            ".beta": ".bias",
        })
        do_not_transpose = do_not_transpose + ["predictions.decoder.weight"]

    pytorch_state_dict = torch.load(pytorch_checkpoint_path, map_location="cpu")
    paddle_state_dict = OrderedDict()
    for k, v in pytorch_state_dict.items():
        is_transpose = False
        if k[-7:] == ".weight":
            # embeddings.weight and LayerNorm.weight do not transpose
            if all(d not in k for d in do_not_transpose):
                if ".embeddings." not in k and ".LayerNorm." not in k:
                    if v.ndim == 2:
                        if 'embeddings' not in k:
                            v = v.transpose(0, 1)
                            is_transpose = True
                        is_transpose = False
        oldk = k

        print(f"Converting: {oldk} => {k} | is_transpose {is_transpose}")
        paddle_state_dict[k] = v.data.numpy()

    paddle.save(paddle_state_dict, paddle_dump_path)

def build_torch_data_pipeline():
    tokenizer = HFBertTokenizer.from_pretrained("bert-base-uncased")

    def preprocess_function(examples):
        result = tokenizer(
            examples["sentence"],
            padding=False,
            max_length=128,
            truncation=True,
            return_token_type_ids=True, )#构建分词器
        if "label" in examples:
            result["labels"] = [examples["label"]]
        return result

    # 加载数据
    dataset_test = Dataset.from_csv("demo_sst2_sentence/demo.tsv", sep="\t")
    dataset_test = dataset_test.map(
        preprocess_function,
        batched=False,
        remove_columns=dataset_test.column_names,
        desc="Running tokenizer on dataset", )#构造数据集
    dataset_test.set_format(
        "np", columns=["input_ids", "token_type_ids", "labels"])
    test_sampler = torch.utils.data.SequentialSampler(dataset_test)
    collate_fn = DataCollatorWithPadding(tokenizer)
    data_loader_test = torch.utils.data.DataLoader(
        dataset_test,
        batch_size=4,
        sampler=test_sampler,
        num_workers=0,
        collate_fn=collate_fn, )#装载分好 batch 的数据
    return dataset_test, data_loader_test

def build_paddle_data_pipeline():
    from paddlenlp.data import DataCollatorWithPadding
    def read(data_path):
        df = pd.read_csv(data_path, sep="\t")
        for _, row in df.iterrows():
            yield {"sentence": row["sentence"], "labels": row["label"]}

    def convert_example(example, tokenizer, max_length=128):
        labels = [example["labels"]]
        #labels = np.array([example["labels"]], dtype="int64")
        example = tokenizer(example["sentence"], max_seq_len=max_length)

        example["labels"] = labels
        return example

    # 加载 tokenizer
    tokenizer = PPNLPBertTokenizer.from_pretrained("bert-base-uncased")#构建分词器
    # 加载 数据集
    dataset_test = ppnlp_load_dataset(
        read, data_path='demo_sst2_sentence/demo.tsv', lazy=False)#构造数据集
    trans_func = partial(convert_example, tokenizer=tokenizer, max_length=128)
    # 使用 tokenize 转换数据
    dataset_test = dataset_test.map(trans_func, lazy=False)
    # 定义 sampler，采样、shuffle 数据
    test_sampler = paddle.io.SequenceSampler(dataset_test)
    test_batch_sampler = paddle.io.BatchSampler(
        sampler=test_sampler, batch_size=4)
    # 定义 数据组合成 batch 的函数
    data_collator = DataCollatorWithPadding(tokenizer)
    # 定义 DataLoader
    data_loader_test = paddle.io.DataLoader(
        dataset_test,
        batch_sampler=test_batch_sampler,
        num_workers=0,
        collate_fn=data_collator, ) #装载分好 batch 的数据
    return dataset_test, data_loader_test

from functools import partial

import numpy as np
import paddle
import pandas as pd
import torch
from datasets import Dataset
from paddlenlp.data import Dict, Pad, Stack
from paddlenlp.datasets import load_dataset as ppnlp_load_dataset
from paddlenlp.transformers import BertTokenizer as PPNLPBertTokenizer
from reprod_log import ReprodDiffHelper, ReprodLogger
from transformers import BertTokenizer as HFBertTokenizer
from transformers import DataCollatorWithPadding

def build_paddle_data_pipeline():
    def read(data_path):
        df = pd.read_csv(data_path, sep="\t")
        for _, row in df.iterrows():
            yield {"sentence": row["sentence"], "labels": row["label"]}
    #数据转换函数
    def convert_example(example, tokenizer, max_length=128):
        labels = np.array([example["labels"]], dtype="int64")
        example = tokenizer(example["sentence"], max_seq_len=max_length)
        return {
            "input_ids": np.array(
                example["input_ids"], dtype="int64"),
            "token_type_ids": np.array(
                example["token_type_ids"], dtype="int64"),
            "labels": labels,
        }

    # 加载分词器
    tokenizer = PPNLPBertTokenizer.from_pretrained("bert-base-uncased")
    # 加载数据
    dataset_test = ppnlp_load_dataset(
        read, data_path='demo_sst2_sentence/demo.tsv', lazy=False)
    trans_func = partial(convert_example, tokenizer=tokenizer, max_length=128)
    # 对数据进行分词
    dataset_test = dataset_test.map(trans_func, lazy=False)
    collate_fn = lambda samples, fn=Dict({
        "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),
        "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
        "labels": Stack(dtype="int64"), }): fn(samples)
    test_sampler = paddle.io.SequenceSampler(dataset_test)
    test_batch_sampler = paddle.io.BatchSampler(
        sampler=test_sampler, batch_size=4)
    data_loader_test = paddle.io.DataLoader(
        dataset_test,
        batch_sampler=test_batch_sampler,
        num_workers=0,
        collate_fn=collate_fn, )

    return dataset_test, data_loader_test


def build_torch_data_pipeline():
    tokenizer = HFBertTokenizer.from_pretrained("bert-base-uncased")
    #数据转换函数
    def preprocess_function(examples):
        result = tokenizer(
            examples["sentence"],
            padding=False,
            max_length=128,
            truncation=True,
            return_token_type_ids=True, )
        if "label" in examples:
            result["labels"] = [examples["label"]]
        return result

    # 加载数据
    dataset_test = Dataset.from_csv("demo_sst2_sentence/demo.tsv", sep="\t")
    dataset_test = dataset_test.map(
        preprocess_function,
        batched=False,
        remove_columns=dataset_test.column_names,
        desc="Running tokenizer on dataset", )
    dataset_test.set_format(
        "np", columns=["input_ids", "token_type_ids", "labels"])
    test_sampler = torch.utils.data.SequentialSampler(dataset_test)
    collate_fn = DataCollatorWithPadding(tokenizer)
    data_loader_test = torch.utils.data.DataLoader(
        dataset_test,
        batch_size=4,
        sampler=test_sampler,
        num_workers=0,
        collate_fn=collate_fn, )
    return dataset_test, data_loader_test


def test_data_pipeline():
    diff_helper = ReprodDiffHelper()
    paddle_dataset, paddle_dataloader = build_paddle_data_pipeline()
    torch_dataset, torch_dataloader = build_torch_data_pipeline()

    logger_paddle_data = ReprodLogger()
    logger_torch_data = ReprodLogger()

    logger_paddle_data.add("length", np.array(len(paddle_dataset)))
    logger_torch_data.add("length", np.array(len(torch_dataset)))

    # 迭代五轮验证
    for idx in range(5):
        rnd_idx = np.random.randint(0, len(paddle_dataset))
        for k in ["input_ids", "token_type_ids", "labels"]:

            logger_paddle_data.add(f"dataset_{idx}_{k}",
                                   paddle_dataset[rnd_idx][k])

            logger_torch_data.add(f"dataset_{idx}_{k}",
                                  torch_dataset[rnd_idx][k])

    for idx, (paddle_batch, torch_batch
              ) in enumerate(zip(paddle_dataloader, torch_dataloader)):
        if idx >= 5:
            break
        for i, k in enumerate(["input_ids", "token_type_ids", "labels"]):
            logger_paddle_data.add(f"dataloader_{idx}_{k}",
                                   paddle_batch[i].numpy())
            logger_torch_data.add(f"dataloader_{idx}_{k}",
                                  torch_batch[k].cpu().numpy())

    diff_helper.compare_info(logger_paddle_data.data, logger_torch_data.data)
    diff_helper.report()


if __name__ == "__main__":
    test_data_pipeline()

no_decay = ["bias", "LayerNorm.weight"]
    optimizer_grouped_parameters = [
        {
            "params": [
                p for n, p in model.named_parameters()
                if not any(nd in n for nd in no_decay)
            ],
            "weight_decay": 1e-2,#参数权重衰减
        },
        {
            "params": [
                p for n, p in model.named_parameters()
                if any(nd in n for nd in no_decay)
            ],
            "weight_decay": 0.0,
        },
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5)

    decay_params = [
        p.name for n, p in model.named_parameters()
        if not any(nd in n for nd in ["bias", "norm"])
    ]
    optimizer = paddle.optimizer.AdamW(
        learning_rate=3e-5,
        parameters=model.parameters(),
        weight_decay=1e-2,#参数权重衰减
        epsilon=1e-6,
        apply_decay_param_fun=lambda x: x in decay_params, )

import sys
import os

import numpy as np
import paddle
import torch
from reprod_log import ReprodLogger
from transformers import AdamW

CURRENT_DIR = os.path.split(os.path.abspath(__file__))[0]  # 当前目录
CONFIG_PATH = CURRENT_DIR.rsplit('/', 1)[0]
sys.path.append(CONFIG_PATH)

from models.pd_bert import (
    BertForSequenceClassification as PDBertForSequenceClassification, )
from models.pd_bert import (
    BertConfig as PDBertConfig, )
from models.pt_bert import (
    BertForSequenceClassification as HFBertForSequenceClassification, )
from models.pt_bert import (
    BertConfig as HFBertConfig, )


def pd_train_some_iters(model,
                        criterion,
                        optimizer,
                        fake_data,
                        fake_label,
                        max_iter=2):
    paddle_dump_path = '../weights/paddle_weight.pdparams'
    config = PDBertConfig()
    model = PDBertForSequenceClassification(config)
    checkpoint = paddle.load(paddle_dump_path)
    model.bert.load_dict(checkpoint)

    classifier_weights = paddle.load(
        "../classifier_weights/paddle_classifier_weights.bin")
    model.load_dict(classifier_weights)
    model.eval()
    criterion = paddle.nn.CrossEntropy()
    decay_params = [
        p.name for n, p in model.named_parameters()
        if not any(nd in n for nd in ["bias", "norm"])
    ]
    optimizer = paddle.optimizer.AdamW(
        learning_rate=3e-5,
        parameters=model.parameters(),
        weight_decay=1e-2,
        epsilon=1e-6,
        apply_decay_param_fun=lambda x: x in decay_params,
    )
    loss_list = []
    for idx in range(max_iter):
        input_ids = paddle.to_tensor(fake_data)
        labels = paddle.to_tensor(fake_label)

        output = model(input_ids)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        optimizer.clear_grad()
        loss_list.append(loss)
    return loss_list


def hf_train_some_iters(fake_data, fake_label, max_iter=2):

    pytorch_dump_path = '../weights/torch_weight.bin'
    config = HFBertConfig()
    model = HFBertForSequenceClassification(config)
    checkpoint = torch.load(pytorch_dump_path)
    model.bert.load_state_dict(checkpoint)
    classifier_weights = torch.load(
        "../classifier_weights/torch_classifier_weights.bin")
    model.load_state_dict(classifier_weights, strict=False)
    model.eval()
    criterion = torch.nn.CrossEntropyLoss()
    no_decay = ["bias", "LayerNorm.weight"]
    optimizer_grouped_parameters = [
        {
            "params": [
                p for n, p in model.named_parameters()
                if not any(nd in n for nd in no_decay)
            ],
            "weight_decay":
            1e-2,
        },
        {
            "params": [
                p for n, p in model.named_parameters()
                if any(nd in n for nd in no_decay)
            ],
            "weight_decay":
            0.0,
        },
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5)

    loss_list = []
    for idx in range(max_iter):
        input_ids = torch.from_numpy(fake_data)
        labels = torch.from_numpy(fake_label)

        output = model(input_ids)[0]
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        loss_list.append(loss)
    return loss_list


if __name__ == "__main__":
    print("Start training")
    paddle.set_device("cpu")
    fake_data = np.load("../fake_data/fake_data.npy")
    fake_label = np.load("../fake_data/fake_label.npy")
    hf_reprod_logger = ReprodLogger()
    hf_loss_list = hf_train_some_iters(fake_data, fake_label, 10)
    for idx, loss in enumerate(hf_loss_list):
        hf_reprod_logger.add(f"loss_{idx}", loss.detach().cpu().numpy())
    hf_reprod_logger.save("bp_align_torch.npy")

    pd_reprod_logger = ReprodLogger()
    pd_loss_list = hf_train_some_iters(fake_data, fake_label, 10)
    for idx, loss in enumerate(pd_loss_list):
        pd_reprod_logger.add(f"loss_{idx}", loss.detach().cpu().numpy())
    pd_reprod_logger.save("bp_align_paddle.npy")