GPU Coder Image output?

I'm using gpu coder to generate the function below, modified from the resnet50_demo, to run a yolo network to a jetson Nano.

function out = resnet50_wrapper(im)
persistent yolov2Obj;
opencv_link_flags = '`pkg-config --cflags --libs opencv`';
coder.updateBuildInfo('addLinkFlags',opencv_link_flags);
if isempty(yolov2Obj)
    yolov2Obj = coder.loadDeepLearningNetwork('C:\Users\josep\Desktop\Fall_2019\Senior_Design\Matlab_Tools\yolov2_5eps_128.mat');
end

% pass in input
% [bboxes,scores,labels] = yolov2Obj.detect(im,'Threshold',0.5);
[bboxes,~,labels] = yolov2Obj.detect(im,'Threshold',0.5);

% Annotate detections in the image.
out = insertObjectAnnotation(im,'rectangle',bboxes,labels);
end

I believe the code generation for the function works fine, but what's the best way to display the output image? I am editting the custom main function from the resnet demo, the function returns the 'out' image as a real32_T, how do I convert it to a mat to display with openCV? The line I'm thinking of adding is just a type conversion underneath "convert out -> oFrame".

/* The MathWorks Inc. 2019*/ 
/* ResNet50 demo main.cu file with OpenCV interfaces to read and display data. */

#include "resnet50_wrapper.h"
#include "main.h"
#include "resnet50_wrapper_terminate.h"
#include "resnet50_wrapper_initialize.h"
#include "opencv2/opencv.hpp"
#include <stdio.h>
#include <stdlib.h>

#define IMG_WIDTH 224
#define IMG_HEIGHT 224
#define IMG_CH 3
#define VID_DEV_ID 1
        
using namespace cv;
using namespace std;
       
static void main_resnet50_wrapper();

/* 
* Convert BGR data to RGB data, without this conversion the predictions 
* will be bad
*/
static void argInit_224x224x3_real32_T(real32_T *input, Mat & im)
{
    for(int j=0;j<224*224;j++)
    {
        //BGR to RGB
        input[2*224*224+j]=(float)(im.data[j*3+0]);
        input[1*224*224+j]=(float)(im.data[j*3+1]);
        input[0*224*224+j]=(float)(im.data[j*3+2]);
    }
}

static void main_resnet50_wrapper(void)
{
  static real32_T b[150528];
  static real32_T out[150528];

  Mat oFrame, cFrame;

  /* Create a Video capture object */
  VideoCapture cap(VID_DEV_ID);
  if(!cap.isOpened())
  {
    cout << "can't open camera" << endl;
    exit(0);
  }
  namedWindow("resnet Demo",CV_WINDOW_NORMAL);
  resizeWindow("resnet Demo", 1000,1000);
  float fps=0;	
  cudaEvent_t start, stop;
  cudaEventCreate(&start);
  cudaEventCreate(&stop);		
  
  while(1)
  {    
    cap >> oFrame;
    resize(oFrame,cFrame,Size(IMG_WIDTH,IMG_HEIGHT));
    
    /* convert from BGR to RGB*/
    argInit_224x224x3_real32_T(b,cFrame);
    cudaEventRecord(start);
    
    /* call the resnet predict  function*/
    // resnet50_wrapper(b, out);
       
    resnet50_wrapper(b,out);
    cudaEventRecord(stop);
    cudaEventSynchronize(stop);
    float milliseconds = -1.0;
    cudaEventElapsedTime(&milliseconds, start, stop);
    fps = fps*.9+1000.0/milliseconds*.1;

    // convert out -> oFrame
    
    // show display        
    imshow("Yolo Demo", oFrame);

    if(waitKey(1)%256 == 27 ) break; // stop when ESC key is pressed
  }  
}

int32_T main(int32_T argc, const char * const argv[])
{
  (void)argc;
  (void)argv;
  
  /* Call the application intialize function */
  resnet50_wrapper_initialize();
  
  /* Call the resnet predict function */
  main_resnet50_wrapper();

 /* Call the application terminate function */
  resnet50_wrapper_terminate();
  return 0;
}

Answer 1

[Dinesh Iyer]

Joseph,

A few things that might help. The cv::Mat's can store data in different color formats. The most popular one that I have seen is BGRA. You can confirm this by inspecting oFrame.channels() and oFrame.depth(). So, your conversion routine argInit_224x224x3_real32_T probably needs to be modified to perform the copy by inspecting the im.channels() and possibly im.depth(). Converting from RGB to cv::Mat is the exact inverse operation. But try to convert it to BGRA.

Hope this helps.

Dinesh