opencv聯合dlib視訊人臉識别例子

本篇文章是在上一篇文章opencv聯合dlib人臉識别例子 的基礎上做了一個實時視訊人臉識别功能。

原理是利用opencv實時提取視訊中的視訊流，然後進入人臉檢測步驟，步驟類似上篇文章。

本篇文章中的程式是在VMware虛拟機下運作的，比較卡，加入人臉識别環節導緻視訊很不流暢。不過本文章中的代碼依舊是一個視訊人臉識别的典型思路的例子。

人臉識别效果圖

工程項目目錄：

linux安裝好opencv和dlib後，解壓工程代碼到linux環境下，進入目錄執行make，執行

./t11 hls_faces huanlesong.mp4

即可運作本例子

點選 這裡 下載下傳本文章工程源代碼。點選無效請通路 https://download.csdn.net/download/u012819339/10667176

代碼以及詳細解釋

#include <dlib/image_processing/frontal_face_detector.h>
#include <dlib/image_processing/render_face_detections.h>
#include <dlib/image_processing.h>
#include <dlib/gui_widgets.h>
#include <dlib/image_io.h>
#include <dlib/opencv.h>
#include <dlib/dnn.h>
#include <dlib/data_io.h>
#include <dlib/clustering.h>
#include <dlib/string.h>

#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc/imgproc.hpp>

#include <iostream>
#include <vector>
#include <ctime>
#include <string>
#include <map>
#include <sstream> 

#ifdef __cplusplus
extern "C"{
#endif

#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>

#ifdef __cplusplus
}
#endif

//由于dlib和opencv中有相當一部分類同名，故不能同時對它們使用using namespace，否則會出現一些莫名其妙的問題
//且dlib庫和标準std庫中的類發生沖突，如map,string 類等等
using namespace std;
using namespace cv;
//using namespace dlib;

void getFiles(std::string path, std::map<std::string, std::string> &files);
void line_one_face_detections(cv::Mat img, std::vector<dlib::full_object_detection> fs);

//定義好一堆模闆别名，以供後續友善使用
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual = dlib::add_prev1<block<N,BN,,dlib::tag1<SUBNET>>>;

template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual_down = dlib::add_prev2<dlib::avg_pool<,,,,dlib::skip1<dlib::tag2<block<N,BN,,dlib::tag1<SUBNET>>>>>>;

template <int N, template <typename> class BN, int stride, typename SUBNET> 
using block  = BN<dlib::con<N,,,,,dlib::relu<BN<dlib::con<N,,,stride,stride,SUBNET>>>>>;

template <int N, typename SUBNET> using ares      = dlib::relu<residual<block,N,dlib::affine,SUBNET>>;
template <int N, typename SUBNET> using ares_down = dlib::relu<residual_down<block,N,dlib::affine,SUBNET>>;

template <typename SUBNET> using alevel0 = ares_down<,SUBNET>;
template <typename SUBNET> using alevel1 = ares<,ares<,ares_down<,SUBNET>>>;
template <typename SUBNET> using alevel2 = ares<,ares<,ares_down<,SUBNET>>>;
template <typename SUBNET> using alevel3 = ares<,ares<,ares<,ares_down<,SUBNET>>>>;
template <typename SUBNET> using alevel4 = ares<,ares<,ares<,SUBNET>>>;

using anet_type = dlib::loss_metric<dlib::fc_no_bias<,dlib::avg_pool_everything<
                            alevel0<
                            alevel1<
                            alevel2<
                            alevel3<
                            alevel4<
                            dlib::max_pool<,,,,dlib::relu<dlib::affine<dlib::con<,,,,,
                            dlib::input_rgb_image_sized<>
                            >>>>>>>>>>>>;




/*
識别視訊中的某一幀圖像中是不是有庫裡的某個人
方法：
統計出庫檔案夾中所有人的圖檔的face_descriptors，然後計算出目前圖檔中的人臉face_descriptors，二者之間距離小于0.6則視為同一個人
./t11 hls_faces huanlesong.mp4
*/

int main(int argc, char *argv[])
{
    time_t start_t, end_t;
    if(argc != )
    {
        std::cout<< "you should specified a dir and a video stream!"<<std::endl;
        return ;
    }
    time(&start_t);
    std::map<string, string> files;
    getFiles(argv[], files);

    if(files.empty())
    {
        std::cout<< "No pic files found in "<< argv[] <<std::endl;
        return ;
    }

    //加載訓練好的級聯分類器，利用haar級聯分類器快速找出人臉區域，然後交給dlib檢測人臉部位
    cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
    //cv::CascadeClassifier faceDetector("./output/cascade.xml");
    if(faceDetector.empty())
    {
        std::cout << "face detector is empty!" <<std::endl;
        return ;
    }

    //加載人臉形狀探測器
    dlib::shape_predictor sp;
    dlib::deserialize("./shape_predictor_68_face_landmarks.dat") >> sp;

    //加載負責人臉識别的DNN
    anet_type net;
    dlib::deserialize("dlib_face_recognition_resnet_model_v1.dat") >> net;

    //人臉描述符庫, face_descriptor ---> name
    map<dlib::matrix<float,0,1>, string> fdlib;

    for(map<string, string>::iterator it = files.begin(); it != files.end(); it++  )
    {
        std::cout << "filename:" << it->second << " filepath:" <<it->first<<std::endl;

        cv::Mat frame = cv::imread(it->first);
        cv::Mat src;
        cv::cvtColor(frame, src, CV_BGR2GRAY);
        dlib::array2d<dlib::bgr_pixel> dimg;
        dlib::assign_image(dimg, dlib::cv_image<uchar>(src)); 

        //haar級聯分類器探測人臉區域，擷取一系列人臉所在區域
        std::vector<cv::Rect> objects;
        std::vector<dlib::rectangle> dets;
        faceDetector.detectMultiScale(src, objects);
        for (int i = ; i < objects.size(); i++)
        {
            //cv::rectangle(frame, objects[i], CV_RGB(200,0,0));
            dlib::rectangle r(objects[i].x, objects[i].y, objects[i].x + objects[i].width, objects[i].y + objects[i].height);
            dets.push_back(r);  //正常情況下應該隻檢測到一副面容
        }

        if (dets.size() == )
            continue;

        std::vector<dlib::matrix<dlib::rgb_pixel>> faces;
        std::vector<dlib::full_object_detection> shapes;
        for(int i = ; i < dets.size(); i++)
        {
            dlib::full_object_detection shape = sp(dimg, dets[i]); //擷取指定一個區域的人臉形狀
            shapes.push_back(shape); 

            dlib::matrix<dlib::rgb_pixel> face_chip;
            dlib::extract_image_chip(dimg, dlib::get_face_chip_details(shape,,), face_chip);

            faces.push_back(move(face_chip));
        }

        if (faces.size() == )
        {
            cout << "No faces found in " << it->second<<endl;
            continue;
        }

        std::vector<dlib::matrix<float,0,1>> face_descriptors = net(faces);

        for(std::vector<dlib::matrix<float,0,1>>::iterator iter = face_descriptors.begin(); iter != face_descriptors.end(); iter++ )
        {
            fdlib.insert(pair<dlib::matrix<float,,>, string>(*iter, it->second));
        }

    }

    time(&end_t);
    std::cout << "ok, all pic in lib had been keep on. use time:"<< end_t - start_t << " s" <<std::endl;


    //加載視訊
    VideoCapture capture(argv[]);
    while(true)
    {
        //加載待檢測的圖檔
        cv::Mat frame;
        capture >> frame;
        if (frame.empty())
            break;

        cv::Mat src;
        cv::cvtColor(frame, src, CV_BGR2GRAY);
        dlib::array2d<dlib::bgr_pixel> dimg;
        dlib::assign_image(dimg, dlib::cv_image<uchar>(src));

        //haar級聯分類器探測人臉區域，擷取一系列人臉所在區域
        std::vector<cv::Rect> objects;
        std::vector<dlib::rectangle> dets;
        faceDetector.detectMultiScale(src, objects);
        for (int i = ; i < objects.size(); i++)
        {
            cv::rectangle(frame, objects[i], CV_RGB(,,));
            dlib::rectangle r(objects[i].x, objects[i].y, objects[i].x + objects[i].width, objects[i].y + objects[i].height);
            dets.push_back(r);  //正常情況下應該隻檢測到一副面容
        }

        if (dets.size() == )
        {
            continue;
        }

        std::vector<dlib::matrix<dlib::rgb_pixel>> faces;
        std::vector<dlib::full_object_detection> shapes;
        for(int i = ; i < dets.size(); i++)
        {
            dlib::full_object_detection shape = sp(dimg, dets[i]); //擷取指定一個區域的人臉形狀
            shapes.push_back(shape); 

            dlib::matrix<dlib::rgb_pixel> face_chip;
            dlib::extract_image_chip(dimg, dlib::get_face_chip_details(shape,,), face_chip);

            faces.push_back(move(face_chip));
        }
        if (faces.size() == )
        {
            continue;
        }
        line_one_face_detections(frame, shapes);

        std::vector<dlib::matrix<float,0,1>> face_descriptors = net(faces);

        //周遊庫，查找相似圖像
        float min_distance = ;
        std::string similar_name = "unknown";
        for(map<dlib::matrix<float,0,1>, string>::iterator it=fdlib.begin(); it != fdlib.end(); it++ )
        {
            float distance = length(it->first - face_descriptors[]);
            if( distance <  )  //應該計算一個最近值
            {
                if( distance <= min_distance)
                {
                    min_distance = distance;
                    similar_name = it->second;
                }
            }
        }

        if(min_distance < )
        {
            float similarity = ( - min_distance) *  / ;
            stringstream strStream; 
            strStream << similar_name << ", " << similarity << '%' << endl;
            string s = strStream.str();
            cv::Point org(objects[].x, objects[].y);
            cv::putText(frame, s, org, cv::FONT_HERSHEY_SIMPLEX, , CV_RGB(, , ));
        }


        cv::imshow("frame", frame);
        //等待10ms，如果從鍵盤輸入的是q、Q、或者是Esc鍵，則退出
        int key = cv::waitKey();
        if (key == 'q' || key == 'Q' || key == )
            break;
    }
    return ;
}



void getFiles(string path, map<string, string> &files)
{
    DIR *dir;
    struct dirent *ptr;
    char base[];

    if(path[path.length()-] != '/')
        path = path + "/";

    if((dir = opendir(path.c_str())) == NULL)
    {
        cout<<"open the dir: "<< path <<"error!" <<endl;
        return;
    }

    while((ptr=readdir(dir)) !=NULL )
    {
        ///current dir OR parrent dir 
        if(strcmp(ptr->d_name,".")== || strcmp(ptr->d_name,"..")==) 
            continue; 
        else if(ptr->d_type == ) //file
        {
            string fn(ptr->d_name);
            string name;
            name = fn.substr(, fn.find_last_of("."));

            string p = path + string(ptr->d_name);
            files.insert(pair<string, string>(p, name));
        }
        else if(ptr->d_type == )    ///link file
        {}
        else if(ptr->d_type == )    ///dir
        {}
    }

    closedir(dir);
    return ;
}


void line_one_face_detections(cv::Mat img, std::vector<dlib::full_object_detection> fs)
{
    int i, j;

    for(j=; j<fs.size(); j++)
    {
        cv::Point p1, p2;

        for(i = ; i<; i++)
        {
            // 下巴到臉頰 0 ~ 16
            //左邊眉毛 17 ~ 21
            //右邊眉毛 21 ~ 26
            //鼻梁     27 ~ 30
            //鼻孔        31 ~ 35
            //左眼        36 ~ 41
            //右眼        42 ~ 47
            //嘴唇外圈  48 ~ 59
            //嘴唇内圈  59 ~ 67
            switch(i)
            {
                case :
                case :
                case :
                case :
                case :
                case :
                case :
                case :
                    i++;
                    break;
                default:
                    break;
            }

            p1.x = fs[j].part(i).x();
            p1.y = fs[j].part(i).y();
            p2.x = fs[j].part(i+).x();
            p2.y = fs[j].part(i+).y();
            cv::line(img, p1, p2, cv::Scalar(,,), );
        }

    }
}