import io
import random
import json
#謠言資料檔案路徑
rumor_class_dirs = os.listdir(target_path+"/Chinese_Rumor_Dataset-master/CED_Dataset/rumor-repost/")
#非謠言資料檔案路徑
non_rumor_class_dirs = os.listdir(target_path+"/Chinese_Rumor_Dataset-master/CED_Dataset/non-rumor-repost/")
original_microblog = target_path+"/Chinese_Rumor_Dataset-master/CED_Dataset/original-microblog/"
#謠言标簽為0，非謠言标簽為1
rumor_label="0"
non_rumor_label="1"
#分别統計謠言資料與非謠言資料的總數
rumor_num = 0
non_rumor_num = 0
all_rumor_list = []
all_non_rumor_list = []
#解析謠言資料
for rumor_class_dir in rumor_class_dirs:
if(rumor_class_dir != '.DS_Store'):
#周遊謠言資料，并解析
with open(original_microblog + rumor_class_dir, 'r') as f:
rumor_content = f.read()
rumor_dict = json.loads(rumor_content)
all_rumor_list.append(rumor_label+"\t"+rumor_dict["text"]+"\n")
rumor_num +=1
#解析非謠言資料
for non_rumor_class_dir in non_rumor_class_dirs:
if(non_rumor_class_dir != '.DS_Store'):
with open(original_microblog + non_rumor_class_dir, 'r') as f2:
non_rumor_content = f2.read()
non_rumor_dict = json.loads(non_rumor_content)
all_non_rumor_list.append(non_rumor_label+"\t"+non_rumor_dict["text"]+"\n")
non_rumor_num +=1
print("謠言資料總量為："+str(rumor_num))
print("非謠言資料總量為："+str(non_rumor_num))
輸出結果如下圖2所示：
【自然語言處理（NLP）】基于LSTM的謠言檢測
(二)、寫入all_data.txt
#全部資料進行亂序後寫入all_data.txt
data_list_path="/home/aistudio/data/"
all_data_path=data_list_path + "all_data.txt"
all_data_list = all_rumor_list + all_non_rumor_list
random.shuffle(all_data_list)
#在生成all_data.txt之前，首先将其清空
with open(all_data_path, 'w') as f:
f.seek(0)
f.truncate()
with open(all_data_path, 'a') as f:
for data in all_data_list:
f.write(data)
(三)、生成資料字典
# 生成資料字典
def create_dict(data_path, dict_path):
with open(dict_path, 'w') as f:
f.seek(0)
f.truncate()
dict_set = set()
# 讀取全部資料
with open(data_path, 'r', encoding='utf-8') as f:
lines = f.readlines()
# 把資料生成一個元組
for line in lines:
content = line.split('\t')[-1].replace('\n', '')
for s in content:
dict_set.add(s)
# 把元組轉換成字典，一個字對應一個數字
dict_list = []
i = 0
for s in dict_set:
dict_list.append([s, i])
i += 1
# 添加未知字元
dict_txt = dict(dict_list)
end_dict = {"<unk>": i}
dict_txt.update(end_dict)
end_dict = {"<pad>": i+1}
dict_txt.update(end_dict)
# 把這些字典儲存到本地中
with open(dict_path, 'w', encoding='utf-8') as f:
f.write(str(dict_txt))
print("資料字典生成完成！")
(四)、資料集劃分
# 建立序列化表示的資料,并按照一定比例劃分訓練資料train_list.txt與驗證資料eval_list.txt
def create_data_list(data_list_path):
#在生成資料之前，首先将eval_list.txt和train_list.txt清空
with open(os.path.join(data_list_path, 'eval_list.txt'), 'w', encoding='utf-8') as f_eval:
f_eval.seek(0)
f_eval.truncate()
with open(os.path.join(data_list_path, 'train_list.txt'), 'w', encoding='utf-8') as f_train:
f_train.seek(0)
f_train.truncate()
with open(os.path.join(data_list_path, 'dict.txt'), 'r', encoding='utf-8') as f_data:
dict_txt = eval（f_data.readlines()[0])
with open(os.path.join(data_list_path, 'all_data.txt'), 'r', encoding='utf-8') as f_data:
lines = f_data.readlines()
i = 0
maxlen = 0
with open(os.path.join(data_list_path, 'eval_list.txt'), 'a', encoding='utf-8') as f_eval,open(os.path.join(data_list_path, 'train_list.txt'), 'a', encoding='utf-8') as f_train:
for line in lines:
words = line.split('\t')[-1].replace('\n', '')
maxlen = max(maxlen, len(words))
label = line.split('\t')[0]
labs = ""
# 每8個 抽取一個資料用于驗證
if i % 8 == 0:
for s in words:
lab = str(dict_txt[s])
labs = labs + lab + ','
labs = labs[:-1]
labs = labs + '\t' + label + '\n'
f_eval.write(labs)
else:
for s in words:
lab = str(dict_txt[s])
labs = labs + lab + ','
labs = labs[:-1]
labs = labs + '\t' + label + '\n'
f_train.write(labs)
i += 1
print("資料清單生成完成！")
print("樣本最長長度：" + str(maxlen))
# 列印前2條訓練資料
vocab = load_vocab(os.path.join(data_root_path, 'dict.txt'))
def ids_to_str(ids):
words = []
for k in ids:
w = list(vocab.keys())[list(vocab.values()).index(int(k))]
words.append(w if isinstance(w, str) else w.decode('ASCII'))
return " ".join(words)
file_path = os.path.join(data_root_path, 'train_list.txt')
with io.open(file_path, "r", encoding='utf8') as fin:
i = 0
for line in fin:
i += 1
cols = line.strip().split("\t")
if len(cols) != 2:
sys.stderr.write("[NOTICE] Error Format Line!")
continue
label = int(cols[1])
wids = cols[0].split(",")
print(str(i)+":")
print('sentence list id is:', wids)
print('sentence list is: ', ids_to_str(wids))
print('sentence label id is:', label)
print('---------------------------------')
if i == 2: break
輸出結果如下圖4所示：
【自然語言處理（NLP）】基于LSTM的謠言檢測
vocab = load_vocab(os.path.join(data_root_path, 'dict.txt'))
class RumorDataset(paddle.io.Dataset):
def __init__(self, data_dir):
self.data_dir = data_dir
self.all_data = []
with io.open(self.data_dir, "r", encoding='utf8') as fin:
for line in fin:
cols = line.strip().split("\t")
if len(cols) != 2:
sys.stderr.write("[NOTICE] Error Format Line!")
continue
label = []
label.append(int(cols[1]))
wids = cols[0].split(",")
if len(wids)>=150:
wids = np.array(wids[:150]).astype('int64')
else:
wids = np.concatenate([wids, [vocab["<pad>"]]*(150-len(wids))]).astype('int64')
label = np.array(label).astype('int64')
self.all_data.append((wids, label))
def __getitem__(self, index):
data, label = self.all_data[index]
return data, label
def __len__(self):
return len(self.all_data)
batch_size = 32
train_dataset = RumorDataset(os.path.join(data_root_path, 'train_list.txt'))
test_dataset = RumorDataset(os.path.join(data_root_path, 'eval_list.txt'))
train_loader = paddle.io.DataLoader(train_dataset, places=paddle.CPUPlace(), return_list=True,
shuffle=True, batch_size=batch_size, drop_last=True)
test_loader = paddle.io.DataLoader(test_dataset, places=paddle.CPUPlace(), return_list=True,
shuffle=True, batch_size=batch_size, drop_last=True)
#check
print('=============train_dataset =============')
for data, label in train_dataset:
print(data)
print(np.array(data).shape)
print(label)
break
print('=============test_dataset =============')
for data, label in test_dataset:
print(data)
print(np.array(data).shape)
print(label)
break
輸出結果如下圖5所示：
【自然語言處理（NLP）】基于LSTM的謠言檢測
二、模型配置
import paddle
from paddle.nn import Conv2D, Linear, Embedding
from paddle import to_tensor
import paddle.nn.functional as F
class RNN(paddle.nn.Layer):
def __init__(self):
super(RNN, self).__init__()
self.dict_dim = vocab["<pad>"]
self.emb_dim = 128
self.hid_dim = 128
self.class_dim = 2
self.embedding = Embedding(
self.dict_dim + 1, self.emb_dim,
sparse=False)
self._fc1 = Linear(self.emb_dim, self.hid_dim)
self.lstm = paddle.nn.LSTM(self.hid_dim, self.hid_dim)
self.fc2 = Linear(19200, self.class_dim)
def forward(self, inputs):
# [32, 150]
emb = self.embedding(inputs)
# [32, 150, 128]
fc_1 = self._fc1(emb)
# [32, 150, 128]
x = self.lstm(fc_1)
x = paddle.reshape(x[0], [0, -1])
x = self.fc2(x)
x = paddle.nn.functional.softmax(x)
return x
rnn = RNN()
paddle.summary(rnn,(32,150),"int64")
