用tflearn建構分類器

爬蟲部分

用爬蟲從百度擷取關鍵詞的圖檔，進行簡單的人工删選後作為訓練用的原始資料。爬蟲代碼可從網上友善找到。

# -*- coding:utf-8 -*-

import re

import requests

import os

 

word = 'something'

download_dir = './pic/'

#建立檔案夾

def mdir():

    if not os.path.exists(download_dir):

        os.mkdir(download_dir)

    if not os.path.exists(download_dir+word):

        os.mkdir(download_dir+word)

    os.chdir(download_dir+word)   

#擷取圖檔的位址

def geturl(page_number):

    url = "http://image.baidu.com/search/avatarjson?tn=resultjsonavatarnew&ie=utf-8&word=" + word + "&cg=girl&rn=60&pn=" +str(page_number)

    html = requests.get(url).text

    pic_url = re.findall('"objURL":"(.*?)",',html,re.S)

    return pic_url

#下載下傳圖檔

def downimage(pic_url, page_number):

    i = 1

    for each in pic_url:

        print(each)

        try:

            pic= requests.get(each, timeout=5)

        except requests.exceptions.ConnectionError:

            print('========Error!CAN NOT DOWMLOAD THIS PICTURE!!!========')

            i -= 1

            continue

        except requests.exceptions.Timeout:

            print("========REQUESTTIMEOUT !!!========")

            i -= 1

        string = word+ str(page_number) + '-' + str(i) + '.jpg'

        fp = open(string,'wb')

        fp.write(pic.content)

        fp.close()

        i += 1   

 

if __name__ == '__main__':

    mdir()

    page_number = 10

    page_count = 60

    while True:

        mPicUrl = geturl(page_count)

        downimage(mPicUrl, page_number)

        page_count += 60

        page_number += 1

           

資料處理部分

将下載下傳好的圖檔進行批處理，先把圖檔大小轉化為神經網絡輸入的尺寸（這裡是(64,64,3)），再生成pkl檔案（當然也可以用其他格式，如tfrecord）。

# -*- coding:utf-8 -*-

import PIL.Image as Image

import numpy as np

import random

import pickle

import os

#改變圖像大小為（64，64）

new_x, new_y = 64, 64

 

def resize(folders_path):

    folders = os.listdir(folders_path)

    for folder in folders:

        files = os.listdir(folders_path +folder)

        i=0

        for file in files:

            #讀取圖檔，PIL.Image庫沒有close方法，會導緻運作出錯，是以用内置的open

            with open(folders_path+ folder + '/' + file ,'rb') as f:

                img = Image.open(f).convert('RGB')

                try:

                    resized =img.resize((new_x, new_y), resample=Image.LANCZOS)

                    resized.save(folders_path +folder + '/resize-' + str(i)+'.jpg', format="jpeg")

                except:

                    print('Get ERROR in '+folders_path +folder + '/' + file)

            #删除原圖

            os.remove(folders_path + folder + '/' + file)

            i += 1

               

#函數調用：生成資料集

def initPKL(imgSet_shuffle, train_or_test):

    imgSet = []

    labels = []

    label_names = []

 

    if train_or_test == 'train':

        set_name = 'trainSet.pkl'

    else:

        set_name = 'testSet.pkl'

 

    for i in imgSet_shuffle:

        imgSet.append(i[0])

        labels.append(i[1])

        label_names.append(i[2])

   

    imgSet = np.array(imgSet)

    labels = np.array(labels)

    label_names = np.array(label_names)

    arr = (imgSet,labels,label_names)

    #寫入檔案

    data = (arr[0],arr[1],arr[2])

    output = open(set_name, 'wb')

    pickle.dump(data, output)

    output.close()

 

def initArr(folders_path):

    i = 0

    imgSet = []

    folders = os.listdir(folders_path)

    for folder in folders:

        #類别個數,幾個0代表幾類

        label = [0,0,0]

        files = os.listdir(folders_path +folder)

        label[i] = 1

        for file in files:

            #讀取圖檔

            img_arr = np.array(Image.open(folders_path+ folder + '/' + file)) / 255

            imgSet.append((img_arr, label,folder))

        i += 1

    return imgSet

 

#将圖檔轉換成數組

train_folders_path= './train/'

test_folders_path= './test/'

 

resize(train_folders_path)

resize(test_folders_path)

 

train_imgSet =initArr(train_folders_path)

test_imgSet =initArr(test_folders_path)

 

#打亂順序

random.shuffle(train_imgSet)

random.shuffle(test_imgSet)

 

train_set_shuffle= np.array(train_imgSet)

test_set_shuffle =np.array(test_imgSet)

 

# 分别生成訓練集和測試集

 

initPKL(train_set_shuffle,'train')

initPKL(test_set_shuffle,'test')

 

#測試生成的資料集

f = open('./trainSet.pkl', 'rb')

x, y, z =pickle.load(f)

f.close()

print(np.shape(x[3]), y[3], z[3])

           

深度學習部分

這裡用比較簡單的AlexNet作為例子，建構了一個小型的神經網絡，要注意輸入和輸出的大小（(64,64,3)和3）。除了訓練，代碼還增加了斷點儲存，模型加載，預判預測等功能。

# -*- coding:utf-8 -*-

import numpy as np

import tflearn

from tflearn.data_utils import shuffle

from tflearn.layers.core import input_data, dropout, fully_connected

from tflearn.layers.normalization importlocal_response_normalization

from tflearn.layers.conv import conv_2d, max_pool_2d

from tflearn.layers.estimatorimport regression

from tflearn.data_preprocessing importImagePreprocessing

from tflearn.data_augmentation import ImageAugmentation

import time

import pickle

 

_loadModel = False

_trainModel = True#False

# 加載資料集

X, Y, Y_name=pickle.load(open("trainSet.pkl", "rb"))

X_test, Y_test,Y_test_name =pickle.load(open("testSet.pkl", "rb"))

# 打亂資料

X, Y, Y_name=shuffle(X, Y, Y_name)

#資料處理

img_prep =ImagePreprocessing()

img_prep.add_featurewise_zero_center()

img_prep.add_featurewise_stdnorm()

# 翻轉、旋轉和模糊效果資料集中的圖檔,來創造一些合成訓練資料.

img_aug =ImageAugmentation()

img_aug.add_random_flip_leftright()

img_aug.add_random_flip_updown()

img_aug.add_random_rotation(max_angle=25.)

img_aug.add_random_blur(sigma_max=3.)

 

# Building'AlexNet'，注意輸入維數

network =input_data(shape=[None, 64, 64, 3])

network =conv_2d(network, 96, 11, strides=4, activation='relu')

network =max_pool_2d(network, 3, strides=2)

network =local_response_normalization(network)

network =conv_2d(network, 256, 5, activation='relu')

network =max_pool_2d(network, 3, strides=2)

network =local_response_normalization(network)

network =conv_2d(network, 384, 3, activation='relu')

network =conv_2d(network, 384, 3, activation='relu')

network =conv_2d(network, 256, 3, activation='relu')

network =max_pool_2d(network, 3, strides=2)

network =local_response_normalization(network)

network =fully_connected(network, 4096, restore=True, activation='tanh')

network =dropout(network, 0.5)

network =fully_connected(network, 4096, restore=True, activation='tanh')

network =dropout(network, 0.5)

network =fully_connected(network, 3, restore=True, activation='softmax')

network =regression(network,

                     optimizer='adam',

                     loss='categorical_crossentropy',

                     learning_rate=0.0001)

 

# 把網絡打包為一個模型對象

model =tflearn.DNN(network,

                    tensorboard_verbose=3,

                    tensorboard_dir = './logs/',

                    best_checkpoint_path = './best_checkpoint/best_classifier.tfl.ckpt')

 

if _loadModel == True:

    model.load("./modelSaved/my_model.tflearn")

    tic = time.time()

    poss = model.predict(X_test)

    toc = time.time()

    print('預測所用時間:%.3fms' % (1000*(toc-tic)))

    print('有%.2f%%的機率是**，有%.2f%%的機率是**，有%.2f%%的機率是**' % (100*poss[0][0], 100*poss[0][1], 100*poss[0][2]))

    print('實際為'+str(Y_test_name[0]))

 

if _trainModel == True:

    model.fit(X, Y,

              n_epoch=1000,

              shuffle=True,

              validation_set=0.1,

              #對訓練資料執行資料分割，10%用于驗證

              show_metric=True,

              batch_size=64,

              snapshot_step=200,

              snapshot_epoch=False,

              run_id='AlexNet')

    model.save("modelSaved/my_model.tflearn")

    print("Networktrained and saved as my_model.tflearn !")
           

訓練結果

在指令行輸入指令如：

tensorboard --logdir=classifier\logs\AlexNet

啟動tensorboard檢視訓練結果

計算圖：

訓練後期，訓練準确率在93~95%之間，驗證準确率約可以保持在98%左右。

代碼運作結束後（或強行終止後），修改代碼：

_loadModel = True

_trainModel = False

使其載入訓練好的模型，對測試圖檔（一張）進行預測。預測結果：