1.线性回归法填补缺失值
#随机生成一个线性回归数据from sklearn.datasets import make_regressionX,Y=make_regression(n_samples=100, n_features=1,n_targets=1,noise=10.5,random_state=1)import matplotlib.pyplot as pltplt.scatter( X, #x坐标 Y, #y坐标);plt.show()123456789
x=np.arange(100)y=3*x+4data={'x':x,'y':y}df=pd.DataFrame(data)df.loc[4:6,"y"]=np.NaN#构造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])123456789
from sklearn import linear_modelregr = linear_model.LinearRegression()regr.fit(X_train, y_train)#构造线性回归模型print('Train Set of Score: %.2f' % regr.score(X_train, y_train))print('Train Set of Score: %.2f' % regr.score(X_test, y_test))#线性回归模型预测缺失值regr.predict(pd.DataFrame(df.loc[4:6,"x"]))1234567
#构建模型的训练集与测试集df=pd.merge(pd.DataFrame(X,columns={'x'}),pd.DataFrame(Y,columns={'y'}), left_index=True,right_index=True)df.loc[4:6,"y"]=np.NaN#构造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])12345678
#构建模型的训练集与测试集df=pd.merge(pd.DataFrame(X,columns={'x'}),pd.DataFrame(Y,columns={'y'}), left_index=True,right_index=True)df.loc[4:6,"y"]=np.NaN#构造缺失值X_train=pd.DataFrame(df.loc[21:100,'x'])y_train=pd.DataFrame(df.loc[21:100,'y'])X_test=pd.DataFrame(df.loc[10:20,'x'])y_test=pd.DataFrame(df.loc[10:20,'y'])12345678
#线性回归模型预测缺失值regr.predict(pd.DataFrame(df.loc[4:6,"x"]))12
2.拉格朗日插值法
#拉格朗日插值法import pandas as pdimport matplotlib.pyplot as pltfrom scipy.interpolate import lagrangedef polyinterp(data,k=5): df1=data.copy() print("原始数据(含缺失值):",'',data) for i in range(len(df1)): if (df1['y'].isnull())[i]: #取数索引范围，向插值前取k个，向后取k个 index_=list(range(i-k, i)) + list(range(i+1, i+1+k))#Series索引不为负数 list0=[j for j in index_ if j in df1['y'].sort_index()] y= df1['y'][list0] #y= df1['y'][list(range(i-k, i)) + list(range(i+1, i+1+k))] y = y[y.notnull()]#索引为负则为缺失值，去掉缺失值 f = lagrange(y.index, list(y)) df1.iloc[i,1] = f(i) print("副本插值后:",'',df1) return(df1)def chart_view(df01,df1): df1.rename(columns={'y': 'New y'}, inplace=True) df01['y'].plot(style='k--') df1['New y'].plot(alpha=0.5) plt.legend(loc='best') plt.show()if __name__=='__main__': x=np.linspace(0,10,11) y=x**3+10 data1=np.vstack((x,y)) df0=pd.DataFrame(data1.T,columns=['x','y']) print(df0) df01=df0.copy()#建立副本 df01.loc[2:3,"y"]=np.NaN#构造缺失值 df1=df01.copy() new_data=polyinterp(df1,5)#插值后 chart_view(df01,new_data)#插值前后绘图12345678910111213141516171819202122232425262728293031323334353637
import numpy as npimport pandas as pdimport matplotlib.pyplot as pltfrom scipy.interpolate import lagrangedef polyinterp(data,k=5): df1=data.copy() print("原始数据(含缺失值):",'',data) for i in range(len(df1)): if (df1['y'].isnull())[i]: #取数索引范围，向插值前取k个，向后取k个 index_=list(range(i-k, i)) + list(range(i+1, i+1+k))#Series索引不为负数 list0=[j for j in index_ if j in df1['y'].sort_index()] y= df1['y'][list0] y = y[y.notnull()]#索引为负则为缺失值，去掉缺失值 f = lagrange(y.index, list(y)) df1.iloc[i,1] = f(i) #print("副本插值后:",'',df1) print("副本插值后:",'',df1[40:]) return(df1)def chart_view(df01,df1): df1.rename(columns={'y': 'New y'}, inplace=True) df01['y'].plot(style='k--') df1['New y'].plot(alpha=0.5) plt.legend(loc='best') plt.show()if __name__=='__main__': df01=pd.read_csv(r'lagra_d1.csv',encoding='gbk') df1=df01.copy() new_data=polyinterp(df1,5)#插值后 chart_view(df01,new_data)#插值前后绘图1234567891011121314151617181920212223242526272829303132
编写打磨课件不易，走过路过别忘记给咱点个赞，对Python感兴趣的可以和我一起交流：【python交流群】：602697820
参考文献：
1.https://blog.csdn.net/shener_m/article/details/81706358
2.https://blog.csdn.net/qq_20011607/article/details/81412985