使用注意力机制的LSTM的机器翻译¶

import paddle
import paddle.nn.functional as F
import re
import numpy as np

print(paddle.__version__)

2.3.0

embedding_size = 128
hidden_size = 256
num_encoder_lstm_layers = 1
en_vocab_size = len(list(en_vocab))
cn_vocab_size = len(list(cn_vocab))
epochs = 20
batch_size = 16

encoder = Encoder()
atten_decoder = AttentionDecoder()

opt = paddle.optimizer.Adam(learning_rate=0.001, 
                            parameters=encoder.parameters()+atten_decoder.parameters())

for epoch in range(epochs):
    print("epoch:{}".format(epoch))

    # shuffle training data
    perm = np.random.permutation(len(train_en_sents))
    train_en_sents_shuffled = train_en_sents[perm]
    train_cn_sents_shuffled = train_cn_sents[perm]
    train_cn_label_sents_shuffled = train_cn_label_sents[perm]

    for iteration in range(train_en_sents_shuffled.shape[0] // batch_size):
        x_data = train_en_sents_shuffled[(batch_size*iteration):(batch_size*(iteration+1))]
        sent = paddle.to_tensor(x_data)
        en_repr = encoder(sent)

        x_cn_data = train_cn_sents_shuffled[(batch_size*iteration):(batch_size*(iteration+1))]
        x_cn_label_data = train_cn_label_sents_shuffled[(batch_size*iteration):(batch_size*(iteration+1))].astype('int64')

        # shape: (batch,  num_layer(=1 here) * num_of_direction(=1 here), hidden_size)
        hidden = paddle.zeros([batch_size, 1, hidden_size])
        cell = paddle.zeros([batch_size, 1, hidden_size])

        loss = paddle.zeros([1])
        # the decoder recurrent loop mentioned above
        for i in range(MAX_LEN + 2):
            cn_word = paddle.to_tensor(x_cn_data[:,i:i+1])
            cn_word_label = paddle.to_tensor(x_cn_label_data[:,i])

            logits, (hidden, cell) = atten_decoder(cn_word, hidden, cell, en_repr)
            step_loss = F.cross_entropy(logits, cn_word_label)
            loss += step_loss

        loss = loss / (MAX_LEN + 2)
        if(iteration % 200 == 0):
            print("iter {}, loss:{}".format(iteration, loss.numpy()))

        loss.backward()
        opt.step()
        opt.clear_grad()

W0509 17:07:41.814532   664 gpu_context.cc:278] Please NOTE: device: 0, GPU Compute Capability: 7.0, Driver API Version: 10.1, Runtime API Version: 10.1
W0509 17:07:41.819046   664 gpu_context.cc:306] device: 0, cuDNN Version: 7.6.


epoch:0
iter 0, loss:[7.6886363]
iter 200, loss:[2.9262748]
iter 400, loss:[3.0081756]
epoch:1
iter 0, loss:[3.2125835]
iter 200, loss:[3.18179]
iter 400, loss:[2.9423902]
epoch:2
iter 0, loss:[2.7851977]
iter 200, loss:[2.7419784]
iter 400, loss:[2.4621239]
epoch:3
iter 0, loss:[2.672283]
iter 200, loss:[2.584266]
iter 400, loss:[2.32677]
epoch:4
iter 0, loss:[2.4052372]
iter 200, loss:[1.8932309]
iter 400, loss:[2.1571693]
epoch:5
iter 0, loss:[2.000717]
iter 200, loss:[1.8644465]
iter 400, loss:[1.8747058]
epoch:6
iter 0, loss:[1.871305]
iter 200, loss:[2.0315902]
iter 400, loss:[2.0112412]
epoch:7
iter 0, loss:[1.7302439]
iter 200, loss:[1.8214658]
iter 400, loss:[1.6249468]
epoch:8
iter 0, loss:[1.7998316]
iter 200, loss:[1.5803907]
iter 400, loss:[1.6613247]
epoch:9
iter 0, loss:[1.2896656]
iter 200, loss:[1.3369302]
iter 400, loss:[1.519732]
epoch:10
iter 0, loss:[1.3795035]
iter 200, loss:[1.2675867]
iter 400, loss:[1.2104226]
epoch:11
iter 0, loss:[1.1157016]
iter 200, loss:[1.2835679]
iter 400, loss:[1.1222675]
epoch:12
iter 0, loss:[1.0207562]
iter 200, loss:[1.1707773]
iter 400, loss:[0.8201]
epoch:13
iter 0, loss:[1.0002316]
iter 200, loss:[0.8215195]
iter 400, loss:[0.9058122]
epoch:14
iter 0, loss:[0.8601862]
iter 200, loss:[0.7515929]
iter 400, loss:[0.7899885]
epoch:15
iter 0, loss:[0.7255517]
iter 200, loss:[0.90155596]
iter 400, loss:[0.76351994]
epoch:16
iter 0, loss:[0.79447305]
iter 200, loss:[0.6634166]
iter 400, loss:[0.84688634]
epoch:17
iter 0, loss:[0.6640573]
iter 200, loss:[0.70083827]
iter 400, loss:[0.62246335]
epoch:18
iter 0, loss:[0.76752484]
iter 200, loss:[0.4401383]
iter 400, loss:[0.4717904]
epoch:19
iter 0, loss:[0.40131176]
iter 200, loss:[0.50756806]
iter 400, loss:[0.48447627]

encoder.eval()
atten_decoder.eval()

num_of_exampels_to_evaluate = 10

indices = np.random.choice(len(train_en_sents),  num_of_exampels_to_evaluate, replace=False)
x_data = train_en_sents[indices]
sent = paddle.to_tensor(x_data)
en_repr = encoder(sent)

word = np.array(
    [[cn_vocab['<bos>']]] * num_of_exampels_to_evaluate
)
word = paddle.to_tensor(word)

hidden = paddle.zeros([num_of_exampels_to_evaluate, 1, hidden_size])
cell = paddle.zeros([num_of_exampels_to_evaluate, 1, hidden_size])

decoded_sent = []
for i in range(MAX_LEN + 2):
    logits, (hidden, cell) = atten_decoder(word, hidden, cell, en_repr)
    word = paddle.argmax(logits, axis=1)
    decoded_sent.append(word.numpy())
    word = paddle.unsqueeze(word, axis=-1)
    
results = np.stack(decoded_sent, axis=1)
for i in range(num_of_exampels_to_evaluate):
    en_input = " ".join(filtered_pairs[indices[i]][0])
    ground_truth_translate = "".join(filtered_pairs[indices[i]][1])
    model_translate = ""
    for k in results[i]:
        w = list(cn_vocab)[k]
        if w != '<pad>' and w != '<eos>':
            model_translate += w
    print(en_input)
    print("true: {}".format(ground_truth_translate))
    print("pred: {}".format(model_translate))

i can barely see you
true: 我幾乎看不到你。
pred: 我幾乎看不到你。
he can also speak russian
true: 他也會說俄語。
pred: 他也能說俄語。
he never drinks alcohol
true: 他從不喝酒。
pred: 他從不喝酒。
you need to get ready
true: 你需要做好准备。
pred: 你需要做好准备。
i subscribe to two newspapers
true: 我訂閱了兩份報紙。
pred: 我訂閱了兩份報紙。
i suppose you re hungry
true: 我猜想您饿了。
pred: 我猜想你饿了。
i just need a break
true: 我只想休息一下
pred: 我只想休息一下
you need to get your eyes checked
true: 你得检查一下眼睛。
pred: 你需要甩掉钱。
i wrote this book
true: 我写了这本书。
pred: 我写了这本书。
i paid 200 in taxes
true: 我付了兩百元含稅。
pred: 我付了兩百元含稅。