#!/usr/bin/env python3
import sqlite3
import pickle
from tqdm import tqdm
import pprint
from nltk.translate.bleu_score import sentence_bleu
data_bath = 'db/conversation.db'
test_samples = []
# 測試集,随機抽取資料n條資料
with sqlite3.connect(data_bath) as conn:
print("Opened database successfully")
cursor = conn.execute("SELECT ask, answer from conversation ORDER BY RANDOM () LIMIT 10001")
rows = cursor.fetchall()
for row in tqdm(rows, desc='creating test labels'):
qa = []
ask = []
answer = []
ask.append(row[])
for i in range(len(row[])):
answer.append(row[][i])
qa.append(ask)
qa.append(answer)
test_samples.append(qa)
# test_samples 永久儲存為pkl檔案
with open('test_10000.pkl', 'wb') as output:
# Pickle dictionary using protocol 0.
data = {'test_samples': test_samples}
pprint.pprint(data['test_samples'])
pickle.dump(data, output)
print('save as test_10000.pkl file success')