Using Kinect with Freenect and OpenCV (C++ version)

freenectopencvmat.cpp

/*
Copyright (C) 2010 Arne Bernin
This code is licensed to you under the terms of the GNU GPL, version 2 or version 3;
see:
http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
http://www.gnu.org/licenses/gpl-3.0.txt
*/
/*
Modified Code
Copyright (C) 2013 Jay Rambhia
This code is licensed to you under the terms of the GNU GPL, version 2 or version 3;
see:
http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
http://www.gnu.org/licenses/gpl-3.0.txt
*/
/*
Makefile for Ubuntu

CXXFLAGS = -O2 -g -Wall -fmessage-length=0 `pkg-config opencv --cflags ` -I /usr/include/libusb-1.0
OBJS = freenectopencvmat.o 
LIBS = `pkg-config opencv --libs` -lfreenect
TARGET = kinectopencv
$(TARGET):$(OBJS)
    $(CXX) -o $(TARGET) $(OBJS) $(LIBS)
all:$(TARGET)
clean:
    rm -f $(OBJS) $(TARGET)
*/

#include <stdio.h>
#include <string.h>
#include <math.h>
#include <libfreenect.h>
#include <pthread.h>
#define CV_NO_BACKWARD_COMPATIBILITY
#include <opencv2/opencv.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>

#define FREENECTOPENCV_WINDOW_D "Depthimage"
#define FREENECTOPENCV_WINDOW_N "Normalimage"
#define FREENECTOPENCV_RGB_DEPTH 3
#define FREENECTOPENCV_DEPTH_DEPTH 1
#define FREENECTOPENCV_RGB_WIDTH 640
#define FREENECTOPENCV_RGB_HEIGHT 480
#define FREENECTOPENCV_DEPTH_WIDTH 640
#define FREENECTOPENCV_DEPTH_HEIGHT 480

using namespace cv;
using namespace std;

Mat depthimg, rgbimg, tempimg, canny_temp, canny_img;

pthread_mutex_t mutex_depth = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t mutex_rgb = PTHREAD_MUTEX_INITIALIZER;
pthread_t cv_thread;
 
// callback for depthimage, called by libfreenect
void depth_cb(freenect_device *dev, void *depth, uint32_t timestamp)
 
{
    Mat depth8;
    Mat mydepth = Mat( FREENECTOPENCV_DEPTH_WIDTH,FREENECTOPENCV_DEPTH_HEIGHT, CV_16UC1, depth);

    mydepth.convertTo(depth8, CV_8UC1, 1.0/4.0);
    pthread_mutex_lock( &mutex_depth );
    memcpy(depthimg.data, depth8.data, 640*480);
    // unlock mutex
    pthread_mutex_unlock( &mutex_depth );
 
}
 
// callback for rgbimage, called by libfreenect
 
void rgb_cb(freenect_device *dev, void *rgb, uint32_t timestamp)
{
 
    // lock mutex for opencv rgb image
    pthread_mutex_lock( &mutex_rgb );
    memcpy(rgbimg.data, rgb, FREENECT_VIDEO_RGB_SIZE);
    // unlock mutex
    pthread_mutex_unlock( &mutex_rgb );
}
 
/*
 * thread for displaying the opencv content
 */
void *cv_threadfunc (void *ptr) {
    depthimg = Mat(FREENECTOPENCV_DEPTH_HEIGHT, FREENECTOPENCV_DEPTH_WIDTH, CV_8UC1);
    rgbimg = Mat(FREENECTOPENCV_RGB_HEIGHT, FREENECTOPENCV_RGB_WIDTH, CV_8UC3);
    tempimg = Mat(FREENECTOPENCV_RGB_HEIGHT, FREENECTOPENCV_RGB_WIDTH, CV_8UC3);
    canny_img = Mat(FREENECTOPENCV_RGB_HEIGHT, FREENECTOPENCV_RGB_WIDTH, CV_8UC1);
    canny_temp = Mat(FREENECTOPENCV_DEPTH_HEIGHT, FREENECTOPENCV_DEPTH_WIDTH, CV_8UC3);

    // use image polling
    while (1)
    {
        //lock mutex for depth image
        pthread_mutex_lock( &mutex_depth );
        Canny(depthimg, canny_temp, 50.0, 200.0, 3);
        cvtColor(depthimg,tempimg,CV_GRAY2BGR);
        cvtColor(tempimg,tempimg,CV_HSV2BGR);

        imshow(FREENECTOPENCV_WINDOW_D,tempimg);
        imshow("Depth Canny", canny_temp);
        //unlock mutex for depth image
        pthread_mutex_unlock( &mutex_depth );

        //lock mutex for rgb image
        pthread_mutex_lock( &mutex_rgb );

        cvtColor(rgbimg,tempimg,CV_BGR2RGB);
        cvtColor(tempimg, canny_img, CV_BGR2GRAY);

        imshow(FREENECTOPENCV_WINDOW_N, tempimg);

        Canny(canny_img, canny_img, 50.0, 200.0, 3);

        imshow("Canny Image", canny_img);
        //unlock mutex
        pthread_mutex_unlock( &mutex_rgb );

        // wait for quit key
        if(cvWaitKey(15) == 27)
            break;

    }
    pthread_exit(NULL);

    return NULL;

}
 
int main(int argc, char **argv)
{
 
    freenect_context *f_ctx;
    freenect_device *f_dev;
 
    int res = 0;
    int die = 0;
    printf("Kinect camera test\n");
 
    if (freenect_init(&f_ctx, NULL) < 0)
    {
        printf("freenect_init() failed\n");
        return 1;
    }
 
    if (freenect_open_device(f_ctx, &f_dev, 0) < 0)
    {
        printf("Could not open device\n");
        return 1;
    }
 
    freenect_set_depth_callback(f_dev, depth_cb);
    freenect_set_video_callback(f_dev, rgb_cb);
    freenect_set_video_format(f_dev, FREENECT_VIDEO_RGB);
 
    // create opencv display thread
    res = pthread_create(&cv_thread, NULL, cv_threadfunc, NULL);
    if (res)
    {
        printf("pthread_create failed\n");
        return 1;
    }
    printf("init done\n");
 
    freenect_start_depth(f_dev);
    freenect_start_video(f_dev);
 
    while(!die && freenect_process_events(f_ctx) >= 0 );
}

Makefile

CXXFLAGS = -O2 -g -Wall -fmessage-length=0 `pkg-config opencv --cflags ` -I /usr/include/libusb-1.0
OBJS = freenectopencvmat.o 
LIBS = `pkg-config opencv --libs` -lfreenect
TARGET = kinectopencv
$(TARGET):$(OBJS)
	$(CXX) -o $(TARGET) $(OBJS) $(LIBS)
all:$(TARGET)
clean:
	rm -f $(OBJS) $(TARGET)

hi
i cant speak good english
my error
error: ‘FREENECT_VIDEO_RGB_SIZE’ was not declared in this scope
error: ‘freenect_set_video_format’ was not declared in this scope
i do
comment
//freenect_set_video_format(f_dev, FREENECT_VIDEO_RGB);
and
i remove
FREENECT_VIDEO_RGB_SIZE
in
memcpy(rgbimg.data, rgb, FREENECT_VIDEO_RGB_SIZE);
and
i write
memcpy(rgbimg.data, rgb, 640*480);
but
rgb image is Half!!!!
pls help me
tnx

Make the following change to ensure that full RGB image is displayed

//Troubleshooting step to ensure full rgb image is displayed
memcpy(rgbimg.data,rgb,(FREENECTOPENCV_RGB_WIDTH+0)*(FREENECTOPENCV_RGB_HEIGHT+950));

Any value more than 950 could corrupt Depth image

CXXFLAGS = -O2 -g -Wall -fmessage-length=0 pkg-config opencv --cflags -I /usr/include/libusb-1.0
OBJS = freenectopencvmat.o
LIBS = pkg-config opencv --libs -lfreenect -lpthread
TARGET = kinectopencv
$(TARGET):$(OBJS)
$(CXX) -o $(TARGET) $(OBJS) $(LIBS)
all:$(TARGET)
clean:
rm -f $(OBJS) $(TARGET)

My CMakeLists.txt after renaming the .cpp file into main.cpp:

cmake_minimum_required(VERSION 3.1)

project( pcv1 LANGUAGES CXX )

find_package( OpenCV REQUIRED )

find_package(libfreenect REQUIRED)

find_package( Threads )

add_library(code
#File_you_want_to_include.cpp
#File_you_want_to_include.h
main.cpp
)

set_target_properties(code PROPERTIES
CXX_STANDARD 11
CXX_STANDARD_REQUIRED YES
CXX_EXTENSIONS NO
)

target_link_libraries(code
PUBLIC
${OpenCV_LIBS}
${FREENECT_LIBRARIES}
${CMAKE_THREAD_LIBS_INIT}
)

add_executable(main
main.cpp
)

set_target_properties(main PROPERTIES
CXX_STANDARD 11
CXX_STANDARD_REQUIRED YES
CXX_EXTENSIONS NO
)

target_link_libraries(main
PRIVATE
code
)