1.線性回歸法填補缺失值
#随機生成一個線性回歸資料from sklearn.datasets import make_regressionX,Y=make_regression(n_samples=100, n_features=1,n_targets=1,noise=10.5,random_state=1)import matplotlib.pyplot as pltplt.scatter( X, #x坐标 Y, #y坐标);plt.show()123456789
x=np.arange(100)y=3*x+4data={'x':x,'y':y}df=pd.DataFrame(data)df.loc[4:6,"y"]=np.NaN#構造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])123456789
from sklearn import linear_modelregr = linear_model.LinearRegression()regr.fit(X_train, y_train)#構造線性回歸模型print('Train Set of Score: %.2f' % regr.score(X_train, y_train))print('Train Set of Score: %.2f' % regr.score(X_test, y_test))#線性回歸模型預測缺失值regr.predict(pd.DataFrame(df.loc[4:6,"x"]))1234567
#構模組化型的訓練集與測試集df=pd.merge(pd.DataFrame(X,columns={'x'}),pd.DataFrame(Y,columns={'y'}), left_index=True,right_index=True)df.loc[4:6,"y"]=np.NaN#構造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])12345678
#構模組化型的訓練集與測試集df=pd.merge(pd.DataFrame(X,columns={'x'}),pd.DataFrame(Y,columns={'y'}), left_index=True,right_index=True)df.loc[4:6,"y"]=np.NaN#構造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])12345678
#線性回歸模型預測缺失值regr.predict(pd.DataFrame(df.loc[4:6,"x"]))12
2.拉格朗日插值法
#拉格朗日插值法import pandas as pdimport matplotlib.pyplot as pltfrom scipy.interpolate import lagrangedef polyinterp(data,k=5): df1=data.copy() print("原始資料(含缺失值):",'',data) for i in range(len(df1)): if (df1['y'].isnull())[i]: #取數索引範圍，向插值前取k個，向後取k個 index_=list(range(i-k, i)) + list(range(i+1, i+1+k))#Series索引不為負數 list0=[j for j in index_ if j in df1['y'].sort_index()] y= df1['y'][list0] #y= df1['y'][list(range(i-k, i)) + list(range(i+1, i+1+k))] y = y[y.notnull()]#索引為負則為缺失值，去掉缺失值 f = lagrange(y.index, list(y)) df1.iloc[i,1] = f(i) print("副本插值後:",'',df1) return(df1)def chart_view(df01,df1): df1.rename(columns={'y': 'New y'}, inplace=True) df01['y'].plot(style='k--') df1['New y'].plot(alpha=0.5) plt.legend(loc='best') plt.show()if __name__=='__main__': x=np.linspace(0,10,11) y=x**3+10 data1=np.vstack((x,y)) df0=pd.DataFrame(data1.T,columns=['x','y']) print(df0) df01=df0.copy()#建立副本 df01.loc[2:3,"y"]=np.NaN#構造缺失值 df1=df01.copy() new_data=polyinterp(df1,5)#插值後 chart_view(df01,new_data)#插值前後繪圖12345678910111213141516171819202122232425262728293031323334353637
import numpy as npimport pandas as pdimport matplotlib.pyplot as pltfrom scipy.interpolate import lagrangedef polyinterp(data,k=5): df1=data.copy() print("原始資料(含缺失值):",'',data) for i in range(len(df1)): if (df1['y'].isnull())[i]: #取數索引範圍，向插值前取k個，向後取k個 index_=list(range(i-k, i)) + list(range(i+1, i+1+k))#Series索引不為負數 list0=[j for j in index_ if j in df1['y'].sort_index()] y= df1['y'][list0] y = y[y.notnull()]#索引為負則為缺失值，去掉缺失值 f = lagrange(y.index, list(y)) df1.iloc[i,1] = f(i) #print("副本插值後:",'',df1) print("副本插值後:",'',df1[40:]) return(df1)def chart_view(df01,df1): df1.rename(columns={'y': 'New y'}, inplace=True) df01['y'].plot(style='k--') df1['New y'].plot(alpha=0.5) plt.legend(loc='best') plt.show()if __name__=='__main__': df01=pd.read_csv(r'lagra_d1.csv',encoding='gbk') df1=df01.copy() new_data=polyinterp(df1,5)#插值後 chart_view(df01,new_data)#插值前後繪圖1234567891011121314151617181920212223242526272829303132
編寫打磨課件不易，走過路過别忘記給咱點個贊，對Python感興趣的可以和我一起交流：【python交流群】：602697820
參考文獻：
1.https://blog.csdn.net/shener_m/article/details/81706358
2.https://blog.csdn.net/qq_20011607/article/details/81412985