傳統車道線檢測項目學習總結（附源碼，複現日志與github連結）

傳統車道線檢測

Advanced_Lane_Detection項目的複現，更換了其中的資料，修改了相應腳本，添加了可視化功能。話不多說，直接上圖顯示效果。

文章目錄

• 傳統車道線檢測
• • 環境要求
  • 項目流程
  • • 一：相機标定
    • 二：ROI操作
    • 三：二值化操作
    • 四：透視變換
    • 五：多項式拟合
    • 六：計算曲率半徑與車輛中心距車道中心偏移量
    • 七：可視化
  • 總結與展望
  • • 總結
    • 展望
• 附：
• 鳴謝

環境要求

• Python 3.5
• Numpy
• OpenCV-Python
• Matplotlib
• Pickle

項目流程

一：相機标定

采用棋盤格對相機進行标定，消除畸變，可采用下列方法（本項目采用第二種）

• 法一：執行calibrate_camera.py腳本獲得内參矩陣與畸變系數存入p檔案

def calibrate_camera():
	# 棋盤格個數及其橫縱内角點個數
	objp_dict = {
		1: (9, 6),
		2: (9, 6),
		3: (9, 6),
		4: (9, 6),
		5: (9, 6),
		6: (9, 6),
		7: (9, 6),
		8: (9, 6),
		9: (9, 6),
		10: (9, 6),
		11: (9, 6),
		12: (9, 6),
		13: (9, 6),
		14: (9, 6),
		15: (9, 6),
		16: (9, 6),
		17: (9, 6),
		18: (9, 6),
		19: (9, 6),
		20: (9, 6),
	}

	objp_list = []
	corners_list = []

	for k in objp_dict:
		nx, ny = objp_dict[k]
		objp = np.zeros((nx*ny,3), np.float32)
		objp[:,:2] = np.mgrid[0:nx, 0:ny].T.reshape(-1,2)
		fname = 'camera_cal/calibration%s.jpg' % str(k)
		img = cv2.imread(fname)
		gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
		# 棋盤格角點
		ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)

		if ret == True:
			objp_list.append(objp)
			corners_list.append(corners)
		else:
			print('Warning: ret = %s for %s' % (ret, fname))

	# 相機标定
	img = cv2.imread('test_images/straight_lines1.jpg')
	img_size = (img.shape[1], img.shape[0])
	ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objp_list, corners_list, img_size,None,None)

	return mtx, dist
           

• 法二：借助Matlab相機标定工具包，傻瓜式操作得到内參矩陣與畸變系數存入p檔案

  具體操作請看Matlab相機标定

  

二：ROI操作

設定mask，隻保留車輛前方ROI區域，以便減小二值化處理的難度

def region_of_interest(img, vertices):
    mask = np.zeros_like(img)
    if len(img.shape) > 2:
        channel_count = img.shape[2]
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255
    cv2.fillPoly(mask, vertices, ignore_mask_color)
    masked_image = cv2.bitwise_and(img, mask)

    return masked_image
           

三：二值化操作

基于圖像的梯度和顔色資訊，進行二值化處理，門檻值自行調整，初步定位車道線所在位置

# SobelX，SobelY梯度資訊
def abs_sobel_thresh(img, orient='x', thresh_min=20, thresh_max=100):
	gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
	if orient == 'x':
		abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
	if orient == 'y':
		abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))

	scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
	binary_output = np.zeros_like(scaled_sobel)
	binary_output[(scaled_sobel >= thresh_min) & (scaled_sobel <= thresh_max)] = 1

	return binary_output

# SobelXY梯度資訊
def mag_thresh(img, sobel_kernel=3, mag_thresh=(30, 100)):
	gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
	sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
	sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
	gradmag = np.sqrt(sobelx**2 + sobely**2)
	scale_factor = np.max(gradmag)/255
	gradmag = (gradmag/scale_factor).astype(np.uint8)
	binary_output = np.zeros_like(gradmag)
	binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1

	return binary_output

# direction資訊
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
	gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
	sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
	sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
	absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
	binary_output =  np.zeros_like(absgraddir)
	binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1

	return binary_output

# S通道資訊
def hls_thresh(img, thresh=(100, 255)):
	hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
	s_channel = hls[:,:,2]
	binary_output = np.zeros_like(s_channel)
	binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1])] = 1

	return binary_output

# 疊加資訊，四組門檻值可以适當調整
def combined_thresh(img):
	abs_bin = abs_sobel_thresh(img, orient='x', thresh_min=35, thresh_max=100)
	mag_bin = mag_thresh(img, sobel_kernel=3, mag_thresh=(30, 255))
	dir_bin = dir_threshold(img, sobel_kernel=15, thresh=(0.7, 1.3))
	hls_bin = hls_thresh(img, thresh=(80, 255))

	combined = np.zeros_like(dir_bin)
	combined[(abs_bin == 1 | ((mag_bin == 1) & (dir_bin == 1))) | hls_bin == 1] = 1

	return combined, abs_bin, mag_bin, dir_bin, hls_bin
           

四：透視變換

運用透視變換，将二值圖像轉換為鳥瞰圖，其中源點與目标點坐标的設定非常重要，以便後期使用多項式拟合進一步精準檢測車道線

def perspective_transform(img):
	img_size = (img.shape[1], img.shape[0])

    # 源點（左下，左上，右上，右下）
	src = np.float32([[310, 1080],[750, 760],[910, 760],[1370, 1080]])
	# 目标點（左下，左上，右上，右下）
	dst = np.float32([[500,1080],[500,0],[1300,0], [1300,1080]])

	# 透射變換矩陣，逆透視變換矩陣
	m = cv2.getPerspectiveTransform(src, dst)
	m_inv = cv2.getPerspectiveTransform(dst, src)

	warped = cv2.warpPerspective(img, m, img_size, flags=cv2.INTER_LINEAR)
	unwarped = cv2.warpPerspective(warped, m_inv, (warped.shape[1], warped.shape[0]), flags=cv2.INTER_LINEAR)

	return warped, unwarped, m, m_inv
           

五：多項式拟合

• 直方圖統計，定位峰值為左右車道線搜尋基點
• 運用滑窗檢測車道線，滑窗的數目，大小，面積設定可調
• 多項式方程 ayy+by+c=x 不斷得到新的搜尋基點
• 可視化操作

# 車道線拟合
def line_fit(binary_warped):
	# 直方圖
	histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
	out_img = (np.dstack((binary_warped, binary_warped, binary_warped))*255).astype('uint8')
	midpoint = np.int(histogram.shape[0]/2)
	leftx_base = np.argmax(histogram[100:midpoint]) + 100
	rightx_base = np.argmax(histogram[midpoint:-100]) + midpoint

	# 滑窗數目
	nwindows = 9
	window_height = np.int(binary_warped.shape[0]/nwindows)
	nonzero = binary_warped.nonzero()
	nonzeroy = np.array(nonzero[0])
	nonzerox = np.array(nonzero[1])
	leftx_current = leftx_base
	rightx_current = rightx_base
	# 滑窗寬度允許變動門檻值
	margin = 100
	# 滑窗面積門檻值
	minpix = 50

	left_lane_inds = []
	right_lane_inds = []

	for window in range(nwindows):
		win_y_low = binary_warped.shape[0] - (window+1)*window_height
		win_y_high = binary_warped.shape[0] - window*window_height

		win_xleft_low = leftx_current - margin
		win_xleft_high = leftx_current + margin
		win_xright_low = rightx_current - margin
		win_xright_high = rightx_current + margin

		cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,255,0), 2)
		cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),(0,255,0), 2)

		good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
		good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]

		left_lane_inds.append(good_left_inds)
		right_lane_inds.append(good_right_inds)

		if len(good_left_inds) > minpix:
			leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
		if len(good_right_inds) > minpix:
			rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

	left_lane_inds = np.concatenate(left_lane_inds)
	right_lane_inds = np.concatenate(right_lane_inds)

	leftx = nonzerox[left_lane_inds]
	lefty = nonzeroy[left_lane_inds]
	rightx = nonzerox[right_lane_inds]
	righty = nonzeroy[right_lane_inds]

	left_fit = np.polyfit(lefty, leftx, 2)
	right_fit = np.polyfit(righty, rightx, 2)

	ret = {}
	ret['left_fit'] = left_fit
	ret['right_fit'] = right_fit
	ret['nonzerox'] = nonzerox
	ret['nonzeroy'] = nonzeroy
	ret['out_img'] = out_img
	ret['left_lane_inds'] = left_lane_inds
	ret['right_lane_inds'] = right_lane_inds

	return ret

           

六：計算曲率半徑與車輛中心距車道中心偏移量

借助像素與米的換算關系，可求出曲率半徑以及車輛中心距車道中心偏移量，進而給出車輛狀态

# 計算曲率半徑
def calc_curve(left_lane_inds, right_lane_inds, nonzerox, nonzeroy):
	# 圖檔高度最大值索引
	y_eval = 1079

	# 圖檔機關像素與米的換算關系
	xm_per_pix = 3.75 / 1100
	ym_per_pix = 30 / 720

	leftx = nonzerox[left_lane_inds]
	lefty = nonzeroy[left_lane_inds]
	rightx = nonzerox[right_lane_inds]
	righty = nonzeroy[right_lane_inds]

	left_fit_cr = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
	right_fit_cr = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)

	left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
	right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])

	return left_curverad, right_curverad

# 計算距車道中心偏移量
def calc_vehicle_offset(undist, left_fit, right_fit):
	bottom_y = undist.shape[0] - 1
	bottom_x_left = left_fit[0]*(bottom_y**2) + left_fit[1]*bottom_y + left_fit[2]
	bottom_x_right = right_fit[0]*(bottom_y**2) + right_fit[1]*bottom_y + right_fit[2]
	vehicle_offset = undist.shape[1]/2 - (bottom_x_left + bottom_x_right)/2
	# 圖檔機關像素與米的換算關系
	xm_per_pix = 3.75/1100
	vehicle_offset *= xm_per_pix

	return vehicle_offset
           

七：可視化

在視訊上方分别實時顯示消除畸變的圖檔，鳥瞰圖檔，多項式拟合圖檔以及曲率半徑，車輛中心距車道中心偏移量

def final_viz(undist, left_fit, right_fit, m_inv, left_curve, right_curve, vehicle_offset):
	ploty = np.linspace(0, undist.shape[0]-1, undist.shape[0])
	left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
	right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

	# 建立圖檔
	color_warp = np.zeros((1080, 1920, 3), dtype='uint8')

	pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
	pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
	pts = np.hstack((pts_left, pts_right))

	cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

	newwarp = cv2.warpPerspective(color_warp, m_inv, (undist.shape[1], undist.shape[0]))
	result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)

	# 圖檔右上角顯示曲率半徑，中心偏移量
	avg_curve = (left_curve + right_curve)/2
	string1 = 'R_mean : %.1f m' % avg_curve
	if left_fit[0] > 0 and avg_curve > 500:
		string2 = "gentle right"
	elif left_fit[0] > 0 and avg_curve <= 500:
		string2 = "hard right"
	elif left_fit[0] < 0 and avg_curve > 500:
		string2 = "gentle left"
	elif left_fit[0] < 0 and avg_curve <= 500:
		string2 = "hard left"
	string3 = 'central offset: %.1f m' % vehicle_offset

	font = cv2.FONT_HERSHEY_SIMPLEX
	cv2.putText(result, string1, (1500, 100), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)
	cv2.putText(result, string2, (1500, 300), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)
	cv2.putText(result, string3, (1500, 200), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)

	# 圖檔上方顯示消除失真圖，鳥瞰圖，車道線檢測圖
	small_undist = cv2.resize(undist, (0, 0), fx=0.2, fy=0.2)
	bird_eye, _, _, _ = perspective_transform(undist)
	small_bird_eye = cv2.resize(bird_eye, (0, 0), fx=0.2, fy=0.2)

	img = cv2.cvtColor(undist, cv2.COLOR_BGR2RGB)
	vertices = np.int32([[(50, 1080), (700, 760), (920, 760), (1400, 1080)]])
	masked_image = region_of_interest(img, vertices)
	cv2.line(masked_image, (50, 1080), (700, 760), (0, 0, 0), 10)
	cv2.line(masked_image, (920, 760), (1400, 1080), (0, 0, 0), 25)
	img, abs_bin, mag_bin, dir_bin, hls_bin = combined_thresh(masked_image)
	binary_warped, binary_unwarped, m, m_inv = perspective_transform(img)
	ret = line_fit(binary_warped)
	out_image = viz2(binary_warped, ret)
	small_out_image = cv2.resize(out_image, (0, 0), fx=0.2, fy=0.2)

	x1 = 0
	y1 = 100
	x2 = small_out_image.shape[0]
	y2 = small_out_image.shape[1]
	y3 = small_out_image.shape[1] * 2
	y4 = small_out_image.shape[1] * 3
	result[x1 + 100:x2 + 100, y1:y2 + 100, :] = small_undist
	result[x1 + 100:x2 + 100, y2 + 200:y3 + 200, :] = small_bird_eye
	result[x1 + 100:x2 + 100, y3 + 300:y4 + 300, :] = small_out_image
	cv2.putText(result, "Undist", (100, 80), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)
	cv2.putText(result, "Bird's Eye", (580, 80), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)
	cv2.putText(result, "Line Search", (1080, 80), font, 0.9, (0, 0, 0), 4, cv2.LINE_AA)

	return result
           

總結與展望

總結

• 傳統圖像處理算法檢測車道線受外界因素影響較大，精度一般
• 這個項目的bug在于車道線檢測偶爾不準，後期曲率半徑的計算會失準，但車輛中心距車道中心的偏移量準确
• 透視變換對于車道線檢測精度提升有很大幫助

展望

• 利用偏移量可進一步實作自動修正方位功能
• 透視變換對于遮擋這一因素的消除十分有效，可應用到其他場合

附：

• 完整項目代碼請看：Github位址
• 複現過程日志請看：複現過程日志 可解決複現過程出現的問題

鳴謝

• 特别鳴謝視覺測控與智能導航實驗室澤陽大神，附上大神Github首頁連結
• 鳴謝臨港六狼團隊所有成員以及張老闆的大力指導