ML之NB、LoR：基于NB和LoR算法對Kaggle IMDB影評資料集(國外類似豆瓣電影)情感分析進行分類

輸出結果

資料集：

https://blog.csdn.net/qq_41185868/article/details/88408004 https://blog.csdn.net/qq_41185868/article/details/88408004

核心代碼

#1、讀取資料并做一些基本的預處理(比如說把評論部分的html标簽去掉等等)

def review_to_wordlist(review):

   '''

   把IMDB的評論轉成詞序列

   review_text = BeautifulSoup(review,"lxml").get_text()  # 去掉HTML标簽，拿到内容

   review_text = re.sub("[^a-zA-Z]"," ", review_text)   # 用正規表達式取出符合規範的部分

   words = review_text.lower().split()   # 小寫化所有的詞，并轉成詞list

   return words   # 傳回words

# 使用pandas讀入訓練和測試csv檔案

train = pd.read_csv('F:/File_Python/Resources/Kaggle Film critic emotion/labeledTrainData.tsv', header=0, delimiter="\t", quoting=3)

test = pd.read_csv('F:/File_Python/Resources/Kaggle Film critic emotion/testData.tsv', header=0, delimiter="\t", quoting=3 )

y_train = train['sentiment']  # 取出情感标簽，positive/褒 或者 negative/貶

train_data = []               # 将訓練和測試資料都轉成詞list

for i in range(0,len(train['review'])):

   train_data.append(" ".join(review_to_wordlist(train['review'][i])))

test_data = []

for i in range(0,len(test['review'])):

   test_data.append(" ".join(review_to_wordlist(test['review'][i])))

print(train_data)

print(y_train)

#2、特征處理：從資料裡面拿到有區分度的特征，采用TF-IDF向量方法

from sklearn.feature_extraction.text import TfidfVectorizer as TFIV

# 初始化TFIV對象，去停用詞，加2元語言模型

tfv = TFIV(min_df=3,  max_features=None, strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}', ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1, stop_words = 'english')

X_all = train_data + test_data  # 合并訓練和測試集以便進行TFIDF向量化操作

len_train = len(train_data)

tfv.fit(X_all)    

X_all = tfv.transform(X_all)

X = X_all[:len_train]   # 恢複成訓練集和測試集部分

X_test = X_all[len_train:]

# 3、利用NB算法，多項式樸素貝葉斯

from sklearn.naive_bayes import MultinomialNB as MNB

model_NB = MNB()

model_NB.fit(X, y_train) #特征資料直接灌進來

MNB(alpha=1.0, class_prior=None, fit_prior=True)

from sklearn.cross_validation import cross_val_score

import numpy as np

print ("多項式貝葉斯分類器20折交叉驗證得分: ", np.mean(cross_val_score(model_NB, X, y_train, cv=20, scoring='roc_auc')))

#4、利用LoR算法

from sklearn.linear_model import LogisticRegression as LR

from sklearn.grid_search import GridSearchCV

# 設定grid search的參數

grid_values = {'C':[30]}  

# 設定打分為roc_auc

model_LR = GridSearchCV(LR(penalty = 'L2', dual = True, random_state = 0), grid_values, scoring = 'roc_auc', cv = 20)

# 資料灌進來

model_LR.fit(X,y_train)

# 20折交叉驗證，開始漫長的等待...

GridSearchCV(cv=20, estimator=LogisticRegression(C=1.0, class_weight=None, dual=True,

            fit_intercept=True, intercept_scaling=1, penalty='L2', random_state=0, tol=0.0001),

       fit_params={}, iid=True, loss_func=None, n_jobs=1,

       param_grid={'C': [30]}, pre_dispatch='2*n_jobs', refit=True,

       score_func=None, scoring='roc_auc', verbose=0)

#輸出結果

print (model_LR.grid_scores_)