From d4779b1bb00fcc7a50e798ab4c166187321253ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20J=C3=A4rvinen?= Date: Wed, 2 Oct 2024 19:15:49 +0200 Subject: [PATCH] - addedd NCNN model support to rtsp_ai_player - added printing of inference FPS - simple AI test bench which can be used to compare models --- misc/opencv-dnn-onnx-inference/.gitignore | 74 ---- .../main-opencv-example.cpp | 387 ----------------- misc/opencv-dnn-onnx-inference/main.cpp | 262 ----------- .../opencv-dnn-onnx-inference.pro | 16 - misc/rtsp_ai_player/aiengine.cpp | 13 + misc/rtsp_ai_player/aiengineconfig.h | 4 +- misc/rtsp_ai_player/aienginertsplistener.cpp | 42 ++ misc/rtsp_ai_player/rtsp_ai_player.pro | 17 +- .../src-ncnn/aiengineinferencencnn.cpp | 408 ++++++++++++++++++ .../src-ncnn/aiengineinferencencnn.h | 31 ++ .../aiengineinferenceonnxruntime.cpp | 6 +- .../src-opi5/aiengineinferenceopi5.cpp | 58 ++- 12 files changed, 555 insertions(+), 763 deletions(-) delete mode 100644 misc/opencv-dnn-onnx-inference/.gitignore delete mode 100644 misc/opencv-dnn-onnx-inference/main-opencv-example.cpp delete mode 100644 misc/opencv-dnn-onnx-inference/main.cpp delete mode 100644 misc/opencv-dnn-onnx-inference/opencv-dnn-onnx-inference.pro create mode 100644 misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.cpp create mode 100644 misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.h diff --git a/misc/opencv-dnn-onnx-inference/.gitignore b/misc/opencv-dnn-onnx-inference/.gitignore deleted file mode 100644 index 4a0b530..0000000 --- a/misc/opencv-dnn-onnx-inference/.gitignore +++ /dev/null @@ -1,74 +0,0 @@ -# This file is used to ignore files which are generated -# ---------------------------------------------------------------------------- - -*~ -*.autosave -*.a -*.core -*.moc -*.o -*.obj -*.orig -*.rej -*.so -*.so.* -*_pch.h.cpp -*_resource.rc -*.qm -.#* -*.*# -core -!core/ -tags -.DS_Store -.directory -*.debug -Makefile* -*.prl -*.app -moc_*.cpp -ui_*.h -qrc_*.cpp -Thumbs.db -*.res -*.rc -/.qmake.cache -/.qmake.stash - -# qtcreator generated files -*.pro.user* -CMakeLists.txt.user* - -# xemacs temporary files -*.flc - -# Vim temporary files -.*.swp - -# Visual Studio generated files -*.ib_pdb_index -*.idb -*.ilk -*.pdb -*.sln -*.suo -*.vcproj -*vcproj.*.*.user -*.ncb -*.sdf -*.opensdf -*.vcxproj -*vcxproj.* - -# MinGW generated files -*.Debug -*.Release - -# Python byte code -*.pyc - -# Binaries -# -------- -*.dll -*.exe - diff --git a/misc/opencv-dnn-onnx-inference/main-opencv-example.cpp b/misc/opencv-dnn-onnx-inference/main-opencv-example.cpp deleted file mode 100644 index fc469c4..0000000 --- a/misc/opencv-dnn-onnx-inference/main-opencv-example.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/** - * @file yolo_detector.cpp - * @brief Yolo Object Detection Sample - * @author OpenCV team - */ - -//![includes] -#include -#include -#include -#include -#include -#include "iostream" -#include "common.hpp" -#include -//![includes] - -using namespace cv; -using namespace cv::dnn; - -void getClasses(std::string classesFile); -void drawPrediction(int classId, float conf, int left, int top, int right, int bottom, Mat& frame); -void yoloPostProcessing( - std::vector& outs, - std::vector& keep_classIds, - std::vector& keep_confidences, - std::vector& keep_boxes, - float conf_threshold, - float iou_threshold, - const std::string& model_name, - const int nc - ); - -std::vector classes; - - -std::string keys = - "{ help h | | Print help message. }" - "{ device | 0 | camera device number. }" - "{ model | onnx/models/yolox_s_inf_decoder.onnx | Default model. }" - "{ yolo | yolox | yolo model version. }" - "{ input i | | Path to input image or video file. Skip this argument to capture frames from a camera. }" - "{ classes | | Optional path to a text file with names of classes to label detected objects. }" - "{ nc | 80 | Number of classes. Default is 80 (coming from COCO dataset). }" - "{ thr | .5 | Confidence threshold. }" - "{ nms | .4 | Non-maximum suppression threshold. }" - "{ mean | 0.0 | Normalization constant. }" - "{ scale | 1.0 | Preprocess input image by multiplying on a scale factor. }" - "{ width | 640 | Preprocess input image by resizing to a specific width. }" - "{ height | 640 | Preprocess input image by resizing to a specific height. }" - "{ rgb | 1 | Indicate that model works with RGB input images instead BGR ones. }" - "{ padvalue | 114.0 | padding value. }" - "{ paddingmode | 2 | Choose one of computation backends: " - "0: resize to required input size without extra processing, " - "1: Image will be cropped after resize, " - "2: Resize image to the desired size while preserving the aspect ratio of original image }" - "{ backend | 0 | Choose one of computation backends: " - "0: automatically (by default), " - "1: Halide language (http://halide-lang.org/), " - "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " - "3: OpenCV implementation, " - "4: VKCOM, " - "5: CUDA }" - "{ target | 0 | Choose one of target computation devices: " - "0: CPU target (by default), " - "1: OpenCL, " - "2: OpenCL fp16 (half-float precision), " - "3: VPU, " - "4: Vulkan, " - "6: CUDA, " - "7: CUDA fp16 (half-float preprocess) }" - "{ async | 0 | Number of asynchronous forwards at the same time. " - "Choose 0 for synchronous mode }"; - -void getClasses(std::string classesFile) -{ - std::ifstream ifs(classesFile.c_str()); - if (!ifs.is_open()) - CV_Error(Error::StsError, "File " + classesFile + " not found"); - std::string line; - while (std::getline(ifs, line)) - classes.push_back(line); -} - -void drawPrediction(int classId, float conf, int left, int top, int right, int bottom, Mat& frame) -{ - rectangle(frame, Point(left, top), Point(right, bottom), Scalar(0, 255, 0)); - - std::string label = format("%.2f", conf); - if (!classes.empty()) - { - CV_Assert(classId < (int)classes.size()); - label = classes[classId] + ": " + label; - } - - int baseLine; - Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); - - top = max(top, labelSize.height); - rectangle(frame, Point(left, top - labelSize.height), - Point(left + labelSize.width, top + baseLine), Scalar::all(255), FILLED); - putText(frame, label, Point(left, top), FONT_HERSHEY_SIMPLEX, 0.5, Scalar()); -} - -void yoloPostProcessing( - std::vector& outs, - std::vector& keep_classIds, - std::vector& keep_confidences, - std::vector& keep_boxes, - float conf_threshold, - float iou_threshold, - const std::string& model_name, - const int nc=80) -{ - // Retrieve - std::vector classIds; - std::vector confidences; - std::vector boxes; - - if (model_name == "yolov8" || model_name == "yolov10" || - model_name == "yolov9") - { - cv::transposeND(outs[0], {0, 2, 1}, outs[0]); - } - - if (model_name == "yolonas") - { - // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84] - Mat concat_out; - // squeeze the first dimension - outs[0] = outs[0].reshape(1, outs[0].size[1]); - outs[1] = outs[1].reshape(1, outs[1].size[1]); - cv::hconcat(outs[1], outs[0], concat_out); - outs[0] = concat_out; - // remove the second element - outs.pop_back(); - // unsqueeze the first dimension - outs[0] = outs[0].reshape(0, std::vector{1, 8400, nc + 4}); - } - - // assert if last dim is 85 or 84 - CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]"); - CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: "); - - for (auto preds : outs) - { - preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85] - for (int i = 0; i < preds.rows; ++i) - { - // filter out non object - float obj_conf = (model_name == "yolov8" || model_name == "yolonas" || - model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at(i, 4) ; - if (obj_conf < conf_threshold) - continue; - - Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols); - double conf; - Point maxLoc; - minMaxLoc(scores, 0, &conf, 0, &maxLoc); - - conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf; - if (conf < conf_threshold) - continue; - - // get bbox coords - float* det = preds.ptr(i); - double cx = det[0]; - double cy = det[1]; - double w = det[2]; - double h = det[3]; - - // [x1, y1, x2, y2] - if (model_name == "yolonas" || model_name == "yolov10"){ - boxes.push_back(Rect2d(cx, cy, w, h)); - } else { - boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h, - cx + 0.5 * w, cy + 0.5 * h)); - } - classIds.push_back(maxLoc.x); - confidences.push_back(static_cast(conf)); - } - } - - // NMS - std::vector keep_idx; - NMSBoxes(boxes, confidences, conf_threshold, iou_threshold, keep_idx); - - for (auto i : keep_idx) - { - keep_classIds.push_back(classIds[i]); - keep_confidences.push_back(confidences[i]); - keep_boxes.push_back(boxes[i]); - } -} - -/** - * @function main - * @brief Main function - */ -int main(int argc, char** argv) -{ - CommandLineParser parser(argc, argv, keys); - parser.about("Use this script to run object detection deep learning networks using OpenCV."); - if (parser.has("help")) - { - parser.printMessage(); - return 0; - } - - CV_Assert(parser.has("model")); - CV_Assert(parser.has("yolo")); - // if model is default, use findFile to get the full path otherwise use the given path - std::string weightPath = "yolov8n.onnx"; - std::string yolo_model = "yolov8"; - int nc = parser.get("nc"); - - float confThreshold = parser.get("thr"); - float nmsThreshold = parser.get("nms"); - //![preprocess_params] - float paddingValue = parser.get("padvalue"); - bool swapRB = parser.get("rgb"); - int inpWidth = parser.get("width"); - int inpHeight = parser.get("height"); - Scalar scale = parser.get("scale"); - Scalar mean = parser.get("mean"); - ImagePaddingMode paddingMode = static_cast(parser.get("paddingmode")); - //![preprocess_params] - - // check if yolo model is valid - if (yolo_model != "yolov5" && yolo_model != "yolov6" - && yolo_model != "yolov7" && yolo_model != "yolov8" - && yolo_model != "yolov10" && yolo_model !="yolov9" - && yolo_model != "yolox" && yolo_model != "yolonas") - CV_Error(Error::StsError, "Invalid yolo model: " + yolo_model); - - // get classes - if (parser.has("classes")) - { - getClasses(findFile(parser.get("classes"))); - } - - // load model - //![read_net] - Net net = readNet(weightPath); - int backend = parser.get("backend"); - net.setPreferableBackend(backend); - net.setPreferableTarget(parser.get("target")); - //![read_net] - - VideoCapture cap; - Mat img; - bool isImage = false; - bool isCamera = false; - - isImage = true; - img = imread("bus.png"); - - /* - // Check if input is given - if (parser.has("input")) - { - String input = parser.get("input"); - // Check if the input is an image - if (input.find(".jpg") != String::npos || input.find(".png") != String::npos) - { - img = imread(findFile(input)); - if (img.empty()) - { - CV_Error(Error::StsError, "Cannot read image file: " + input); - } - isImage = true; - } - else - { - cap.open(input); - if (!cap.isOpened()) - { - CV_Error(Error::StsError, "Cannot open video " + input); - } - isCamera = true; - } - } - else - { - int cameraIndex = parser.get("device"); - cap.open(cameraIndex); - if (!cap.isOpened()) - { - CV_Error(Error::StsError, cv::format("Cannot open camera #%d", cameraIndex)); - } - isCamera = true; - } - */ - - // image pre-processing - //![preprocess_call] - Size size(inpWidth, inpHeight); - Image2BlobParams imgParams( - scale, - size, - mean, - swapRB, - CV_32F, - DNN_LAYOUT_NCHW, - paddingMode, - paddingValue); - - // rescale boxes back to original image - Image2BlobParams paramNet; - paramNet.scalefactor = scale; - paramNet.size = size; - paramNet.mean = mean; - paramNet.swapRB = swapRB; - paramNet.paddingmode = paddingMode; - //![preprocess_call] - - //![forward_buffers] - std::vector outs; - std::vector keep_classIds; - std::vector keep_confidences; - std::vector keep_boxes; - std::vector boxes; - //![forward_buffers] - - Mat inp; - while (waitKey(1) < 0) - { - - if (isCamera) - cap >> img; - if (img.empty()) - { - std::cout << "Empty frame" << std::endl; - waitKey(); - break; - } - //![preprocess_call_func] - inp = blobFromImageWithParams(img, imgParams); - //![preprocess_call_func] - - //![forward] - net.setInput(inp); - net.forward(outs, net.getUnconnectedOutLayersNames()); - //![forward] - - //![postprocess] - yoloPostProcessing( - outs, keep_classIds, keep_confidences, keep_boxes, - confThreshold, nmsThreshold, - yolo_model, - nc); - //![postprocess] - - // covert Rect2d to Rect - //![draw_boxes] - for (auto box : keep_boxes) - { - boxes.push_back(Rect(cvFloor(box.x), cvFloor(box.y), cvFloor(box.width - box.x), cvFloor(box.height - box.y))); - } - - paramNet.blobRectsToImageRects(boxes, boxes, img.size()); - - for (size_t idx = 0; idx < boxes.size(); ++idx) - { - Rect box = boxes[idx]; - drawPrediction(keep_classIds[idx], keep_confidences[idx], box.x, box.y, - box.width + box.x, box.height + box.y, img); - } - - const std::string kWinName = "Yolo Object Detector"; - namedWindow(kWinName, WINDOW_NORMAL); - imshow(kWinName, img); - //![draw_boxes] - - outs.clear(); - keep_classIds.clear(); - keep_confidences.clear(); - keep_boxes.clear(); - boxes.clear(); - - if (isImage) - { - waitKey(); - break; - } - } -} diff --git a/misc/opencv-dnn-onnx-inference/main.cpp b/misc/opencv-dnn-onnx-inference/main.cpp deleted file mode 100644 index 560df73..0000000 --- a/misc/opencv-dnn-onnx-inference/main.cpp +++ /dev/null @@ -1,262 +0,0 @@ -#include -#include -#include -#include -#include - -#define CONFIDENCE_THRESHOLD 0.2 - -using namespace std; -using namespace cv; - -// function to create vector of class names -std::vector createClaseNames() { - std::vector classNames; - classNames.push_back("background"); - classNames.push_back("aeroplane"); - classNames.push_back("bicycle"); - classNames.push_back("bird"); - classNames.push_back("boat"); - classNames.push_back("bottle"); - classNames.push_back("bus"); - classNames.push_back("car"); - classNames.push_back("cat"); - classNames.push_back("chair"); - classNames.push_back("cow"); - classNames.push_back("diningtable"); - classNames.push_back("dog"); - classNames.push_back("horse"); - classNames.push_back("motorbike"); - classNames.push_back("person"); - classNames.push_back("pottedplant"); - classNames.push_back("sheep"); - return classNames; -}; - -int main(int argc, char *argv[]) -{ - if (argc < 3) { - qDebug() << "Give ONNX model as first argument and image as second one."; - return 1; - } - std::vector classNames = createClaseNames(); - - std::string model_filename = argv[1]; - std::string image_filename = argv[2]; - - // Load the ONNX model - cv::dnn::Net net = cv::dnn::readNetFromONNX(model_filename); - - // Set backend to OpenCL - net.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT); - //net.setPreferableTarget(cv::dnn::DNN_TARGET_OPENCL); - - // Load an image and preprocess it - cv::Mat image = cv::imread(image_filename); - cv::Mat blob = cv::dnn::blobFromImage(image, 1/255.0, cv::Size(640, 640), cv::Scalar(), true, false); - cv::imwrite("tmpInput.png", image); - - // Set the input for the network - net.setInput(blob); - std::vector outputs; - net.forward(outputs); - cv::Mat output = outputs[0]; - - std::vector layerNames = net.getLayerNames(); - std::vector outLayerIndices = net.getUnconnectedOutLayers(); - for (int index : outLayerIndices) { - std::cout << layerNames[index - 1] << std::endl; - } - - - cv::Mat detections = net.forward("output0"); - - // print some information about detections - std::cout << "dims: " << outputs[0].dims << std::endl; - std::cout << "size: " << outputs[0].size << std::endl; - - int num_detections0 = outputs.size(); - int num_classes0 = outputs[0].size[1]; - int rows = outputs[0].size[1]; - qDebug() << "num_detections0:" << num_detections0 << "num_classes0:" << num_classes0 << "rows:" << rows; - - int num_detections = detections.size[2]; // 8400 - int num_attributes = detections.size[1]; // 14 - qDebug() << "num_detections:" << num_detections << "num_attributes:" << num_attributes; - - - detections = detections.reshape(1, num_detections); - - for (int i = 0; i < num_detections; i++) { - Mat row = detections.row(i); - float box_confidence = row.at(4); - float cls_confidence = row.at(5); - if (box_confidence > 0.5 && cls_confidence > 0.5) { - std::cout << "box_confidence " << box_confidence << " cls_confidence: " << cls_confidence << std::endl; - } - } - - - - - /* - // Output tensor processing - // Assuming output tensor has shape [1, 14, 8400] - int num_detections = detections.size[2]; // 8400 - int num_attributes = detections.size[1]; // 14 - qDebug() << "num_detections:" << num_detections << "num_attributes:" << num_attributes; - - // Extract and print confidence for each detection - const float* data = (float*)output.data; - for (int i = 0; i < num_detections; i++) { - // Confidence value is at index 4 (based on YOLO-like model output format) - float confidence = data[i * num_attributes + 3]; - // Print confidence value - if (confidence > 0.5f) { - std::cout << "Detection " << i << " confidence: " << confidence << std::endl; - } - } - */ - - /* - // Assuming outputs is a vector of cv::Mat containing the model's output - for (int i = 0; i < outputs[0].size[0]; ++i) { - for (int j = 0; j < outputs[0].size[1]; ++j) { - float confidence = outputs[0].at(i, j); - // Print or use the confidence value - std::cout << "Confidence: " << confidence << std::endl; - } - } - */ - - /* - // Output processing variables - std::vector class_ids; - std::vector confidences; - std::vector boxes; - - // Analyze each output - for (const auto& output : outputs) { - // Each row is a detection: [center_x, center_y, width, height, conf, class1_score, class2_score, ...] - const auto* data = (float*)output.data; - - for (int i = 0; i < output.rows; i++, data += output.cols) { - float confidence = data[4]; // Objectness confidence - - if (confidence >= 0.5) { // Apply a confidence threshold - cv::Mat scores = output.row(i).colRange(5, output.cols); - cv::Point class_id_point; - double max_class_score; - - cv::minMaxLoc(scores, 0, &max_class_score, 0, &class_id_point); - int class_id = class_id_point.x; - - if (max_class_score >= 0.5) { // Apply a class score threshold - // Get bounding box - int center_x = (int)(data[0] * image.cols); - int center_y = (int)(data[1] * image.rows); - int width = (int)(data[2] * image.cols); - int height = (int)(data[3] * image.rows); - int left = center_x - width / 2; - int top = center_y - height / 2; - - // Store the results - class_ids.push_back(class_id); - confidences.push_back(confidence); - boxes.push_back(cv::Rect(left, top, width, height)); - } - } - } - } - */ - - /* - for (int i = 0; i < num_detections; ++i) { - for (int a = 0; a < 10; a++) { - qDebug() << "confidence:" << confidence << "class_id" << class_id; - } - //float confidence = outputs[0].at(i, 4); - //int class_id = outputs[0].at(i, 5); - qDebug() << "confidence:" << confidence << "class_id" << class_id; - } - */ - - /* - std::vector class_ids; - std::vector confidences; - std::vector boxes; - // Output tensor processing - for (size_t i = 0; i < outputs.size(); i++) { - float* data = (float*)outputs[i].data; - for (int j = 0; j < outputs[i].rows; ++j) { - float confidence = data[4]; // Objectness confidence - qDebug() <<" Confidence: " << confidence; - - if (confidence >= CONFIDENCE_THRESHOLD) { - // Get class with the highest score - cv::Mat scores = outputs[i].row(j).colRange(5, num_classes + 5); - cv::Point class_id_point; - double max_class_score; - cv::minMaxLoc(scores, 0, &max_class_score, 0, &class_id_point); - - qDebug() << "Class ID: " << class_id_point.x << " Confidence: " << confidence; - - if (max_class_score > CONFIDENCE_THRESHOLD) { - int center_x = (int)(data[0] * image.cols); - int center_y = (int)(data[1] * image.rows); - int width = (int)(data[2] * image.cols); - int height = (int)(data[3] * image.rows); - int left = center_x - width / 2; - int top = center_y - height / 2; - - // Store the results - class_ids.push_back(class_id_point.x); - confidences.push_back((float)max_class_score); - boxes.push_back(cv::Rect(left, top, width, height)); - } - } - data += num_classes + 5; // Move to the next detection - } - } - */ - - /* - // Process the output - for (int i = 0; i < num_detections; ++i) { - float confidence = outputs[0].at(i, 4); - int class_id = outputs[0].at(i, 5); - // ... extract bounding box coordinates - float x_center = outputs[0].at(i, 0); - float y_center = outputs[0].at(i, 1); - float width = outputs[0].at(i, 2); - float height = outputs[0].at(i, 3); - - // Calculate bounding box corners - int x = (int)(x_center - width / 2.0); - int y = (int)(y_center - height / 2.0); - int width_int = (int)width; - int height_int = (int)height; - - // Print or use the detected information - qDebug() << "Class ID: " << class_id << " Confidence: " << confidence << " Bounding Box: (" << x << ", " << y << ", " << width_int << ", " << height_int << ")"; - } - */ - - - /* - // Perform forward pass - for (int i = 0; i < 10; i++) { - std::vector outputs; - QElapsedTimer timer; - timer.start(); - net.forward(outputs); - qDebug() << "Inference completed in" << timer.elapsed() << "ms."; - } - */ - - // Process the output (YOLO-specific post-processing like NMS is needed) - // Outputs contain class scores, bounding boxes, etc. - - - return 0; -} diff --git a/misc/opencv-dnn-onnx-inference/opencv-dnn-onnx-inference.pro b/misc/opencv-dnn-onnx-inference/opencv-dnn-onnx-inference.pro deleted file mode 100644 index 246e760..0000000 --- a/misc/opencv-dnn-onnx-inference/opencv-dnn-onnx-inference.pro +++ /dev/null @@ -1,16 +0,0 @@ -QT = core -QT -= gui -CONFIG += c++17 cmdline concurrent console -# link_pkgconfig - -SOURCES += \ - main-opencv-example.cpp - #main.cpp - -HEADERS += \ - common.hpp - - -INCLUDEPATH += /opt/opencv-4.10.0/include/opencv4/ -LIBS += /opt/opencv-4.10.0/lib/libopencv_world.so -QMAKE_LFLAGS += -Wl,-rpath,/opt/opencv-4.10.0/lib diff --git a/misc/rtsp_ai_player/aiengine.cpp b/misc/rtsp_ai_player/aiengine.cpp index d4357c4..b33d054 100644 --- a/misc/rtsp_ai_player/aiengine.cpp +++ b/misc/rtsp_ai_player/aiengine.cpp @@ -1,5 +1,6 @@ #include #include + #include "aiengine.h" #include "aiengineinference.h" #include "aiengineimagesaver.h" @@ -8,10 +9,14 @@ #include "src-opi5/aiengineinferenceopi5.h" #elif defined(OPENCV_BUILD) #include "src-opencv-onnx/aiengineinferenceopencvonnx.h" +#elif defined(NCNN_BUILD) +#include "src-ncnn/aiengineinferencencnn.h" #else #include "src-onnx-runtime/aiengineinferenceonnxruntime.h" #endif + + AiEngine::AiEngine(QString modelPath, QObject *parent) : QObject{parent} { @@ -27,6 +32,8 @@ AiEngine::AiEngine(QString modelPath, QObject *parent) mInference3->initialize(2); #elif defined(OPENCV_BUILD) mInference = new AiEngineInferenceOpencvOnnx(modelPath); +#elif defined(NCNN_BUILD) + mInference = new AiEngineInferencevNcnn(modelPath); #else mInference = new AiEngineInferencevOnnxRuntime(modelPath); #endif @@ -76,6 +83,7 @@ void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult result) { mFrameCounter++; qDebug() << "FPS = " << (mFrameCounter / (mElapsedTimer.elapsed()/1000.0f)); + //qDebug() << "DEBUG. inference frame counter:" << mFrameCounter; //qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId(); if (mGimbalClient != nullptr) { @@ -97,19 +105,24 @@ void AiEngine::frameReceivedSlot(cv::Mat frame) { //qDebug() << "AiEngine got frame from RTSP listener in thread: " << QThread::currentThreadId(); //cv::imshow("Received Frame", frame); + static int framecounter = 0; + //qDebug() << "DEBUG. RTSP frame counter:" << framecounter; if (mInference->isActive() == false) { //qDebug() << "AiEngine. Inference thread is free. Sending frame to it."; emit inferenceFrame(frame); + framecounter++; } #ifdef OPI5_BUILD else if (mInference2->isActive() == false) { //qDebug() << "AiEngine. Inference thread is free. Sending frame to it."; emit inferenceFrame2(frame); + framecounter++; } else if (mInference3->isActive() == false) { //qDebug() << "AiEngine. Inference thread is free. Sending frame to it."; emit inferenceFrame3(frame); + framecounter++; } #endif } diff --git a/misc/rtsp_ai_player/aiengineconfig.h b/misc/rtsp_ai_player/aiengineconfig.h index 196845a..d724696 100644 --- a/misc/rtsp_ai_player/aiengineconfig.h +++ b/misc/rtsp_ai_player/aiengineconfig.h @@ -3,10 +3,8 @@ #include #ifdef OPI5_BUILD -QString rtspVideoUrl = "rtsp://192.168.0.1:8554/live.stream"; +QString rtspVideoUrl = "rtsp://192.168.168.91:8554/live.stream"; #else -// Video file from the local MTX RTSP server or gimbal camera. QString rtspVideoUrl = "rtsp://localhost:8554/live.stream"; -//QString rtspVideoUrl = "rtsp://192.168.0.25:8554/main.264"; #endif diff --git a/misc/rtsp_ai_player/aienginertsplistener.cpp b/misc/rtsp_ai_player/aienginertsplistener.cpp index bbd3510..47ec74c 100644 --- a/misc/rtsp_ai_player/aienginertsplistener.cpp +++ b/misc/rtsp_ai_player/aienginertsplistener.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include "aienginertsplistener.h" #include "aiengineconfig.h" @@ -39,6 +41,45 @@ void AiEngineRtspListener::stopListening() void AiEngineRtspListener::listenLoop(void) { +#ifdef AI_BENCH + QThread::msleep(2000); + + QString directoryPath = "/home/pama/tmp/photos"; + QDir directory(directoryPath); + + // Ensure the directory exists + if (!directory.exists()) { + qDebug() << "Directory does not exist!"; + exit(1); + } + + QStringList filters; + filters << "*.jpg" << "*.jpeg" << "*.png" << "*.bmp"; + directory.setNameFilters(filters); + + QFileInfoList fileInfoList = directory.entryInfoList(QDir::Files, QDir::Name); + std::sort(fileInfoList.begin(), fileInfoList.end(), [](const QFileInfo &a, const QFileInfo &b) { + return a.fileName() < b.fileName(); + }); + + qDebug() << "Images in folder:" << fileInfoList.size(); + + for (const QFileInfo &fileInfo : fileInfoList) { + QString filePath = fileInfo.absoluteFilePath(); + cv::Mat image = cv::imread(filePath.toStdString()); + if (image.empty()) { + qDebug() << "Failed to load image"; + exit(1); + } + + emit frameReceived(image.clone()); + QThread::msleep(1500); + } + + QThread::msleep(2000); + qDebug() << "Sleep 2000ms and exit"; + exit(0); +#else qDebug() << "AiEngineRtspListener loop running in thread: " << QThread::currentThreadId(); mCap.open(rtspVideoUrl.toStdString()); @@ -51,4 +92,5 @@ void AiEngineRtspListener::listenLoop(void) emit frameReceived(frame.clone()); } } +#endif } diff --git a/misc/rtsp_ai_player/rtsp_ai_player.pro b/misc/rtsp_ai_player/rtsp_ai_player.pro index 1c2366b..8f25d60 100644 --- a/misc/rtsp_ai_player/rtsp_ai_player.pro +++ b/misc/rtsp_ai_player/rtsp_ai_player.pro @@ -1,12 +1,16 @@ QT += core network serialport QT -= gui -CONFIG += c++11 concurrent console +CONFIG += concurrent console c++17 MOC_DIR = moc OBJECTS_DIR = obj SOURCES = $$PWD/*.cpp HEADERS = $$PWD/*.h +ai_bench { + QMAKE_CXXFLAGS += -DAI_BENCH +} + gimbal { message("Using real gimbal camera.") QMAKE_CXXFLAGS += -DGIMBAL @@ -35,6 +39,17 @@ opi5 { SOURCES += $$PWD/src-opi5/*.c $$PWD/src-opi5/*.cpp $$PWD/src-opi5/*.cc HEADERS += $$PWD/src-opi5/*.h } +else:ncnn { + message("NCNN build") + CONFIG += link_pkgconfig + PKGCONFIG += opencv4 + QMAKE_CXXFLAGS += -DNCNN_BUILD -fopenmp + QMAKE_LFLAGS += -fopenmp + INCLUDEPATH += /opt/ncnn/include + LIBS += /opt/ncnn/lib/libncnn.a -lgomp + SOURCES += $$PWD/src-ncnn/*.cpp + HEADERS += $$PWD/src-ncnn/*.h +} else:opencv { message("OpenCV build") message("You must use YOLOv8 ONNX files. Azaion model does not work!") diff --git a/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.cpp b/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.cpp new file mode 100644 index 0000000..edd303a --- /dev/null +++ b/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.cpp @@ -0,0 +1,408 @@ +#include +#include +#include +#include +#include "aiengineinferencencnn.h" + + +const int target_size = 640; +const float prob_threshold = 0.25f; +const float nms_threshold = 0.45f; +const int num_labels = 10; // 80 object labels in COCO, 10 in Azaion +const int MAX_STRIDE = 32; + + +char* getCharPointerCopy(const QString& modelPath) { + QByteArray byteArray = modelPath.toUtf8(); + char* cString = new char[byteArray.size() + 1]; // Allocate memory + std::strcpy(cString, byteArray.constData()); // Copy the data + return cString; // Remember to delete[] this when done! +} + + +AiEngineInferencevNcnn::AiEngineInferencevNcnn(QString modelPath, QObject *parent) : + AiEngineInference{modelPath, parent} +{ + qDebug() << "TUOMAS AiEngineInferencevNcnn() mModelPath=" << mModelPath; + + yolov8.opt.num_threads = 4; + yolov8.opt.use_vulkan_compute = false; + + QString paramPath = modelPath.chopped(3).append("param"); + char *model = getCharPointerCopy(modelPath); + char *param = getCharPointerCopy(paramPath); + + qDebug() << "model:" << model; + qDebug() << "param:" << param; + + yolov8.load_param(param); + yolov8.load_model(model); +} + + +static inline float intersection_area(const Object& a, const Object& b) +{ + cv::Rect_ inter = a.rect & b.rect; + return inter.area(); +} + + +static void qsort_descent_inplace(std::vector& objects, int left, int right) +{ + int i = left; + int j = right; + float p = objects[(left + right) / 2].prob; + + while (i <= j) + { + while (objects[i].prob > p) + i++; + + while (objects[j].prob < p) + j--; + + if (i <= j) + { + // swap + std::swap(objects[i], objects[j]); + + i++; + j--; + } + } + +#pragma omp parallel sections + { +#pragma omp section + { + if (left < j) qsort_descent_inplace(objects, left, j); + } +#pragma omp section + { + if (i < right) qsort_descent_inplace(objects, i, right); + } + } +} + + +static void qsort_descent_inplace(std::vector& objects) +{ + if (objects.empty()) + return; + + qsort_descent_inplace(objects, 0, objects.size() - 1); +} + + +static void nms_sorted_bboxes(const std::vector& faceobjects, std::vector& picked, float nms_threshold, bool agnostic = false) +{ + picked.clear(); + + const int n = faceobjects.size(); + + std::vector areas(n); + for (int i = 0; i < n; i++) + { + areas[i] = faceobjects[i].rect.area(); + } + + for (int i = 0; i < n; i++) + { + const Object& a = faceobjects[i]; + + int keep = 1; + for (int j = 0; j < (int)picked.size(); j++) + { + const Object& b = faceobjects[picked[j]]; + + if (!agnostic && a.label != b.label) + continue; + + // intersection over union + float inter_area = intersection_area(a, b); + float union_area = areas[i] + areas[picked[j]] - inter_area; + // float IoU = inter_area / union_area + if (inter_area / union_area > nms_threshold) + keep = 0; + } + + if (keep) + picked.push_back(i); + } +} + + +static inline float sigmoid(float x) +{ + return static_cast(1.f / (1.f + exp(-x))); +} + + +static inline float clampf(float d, float min, float max) +{ + const float t = d < min ? min : d; + return t > max ? max : t; +} + + +static void parse_yolov8_detections( + float* inputs, float confidence_threshold, + int num_channels, int num_anchors, int num_labels, + int infer_img_width, int infer_img_height, + std::vector& objects) +{ + std::vector detections; + cv::Mat output = cv::Mat((int)num_channels, (int)num_anchors, CV_32F, inputs).t(); + + for (int i = 0; i < num_anchors; i++) + { + const float* row_ptr = output.row(i).ptr(); + const float* bboxes_ptr = row_ptr; + const float* scores_ptr = row_ptr + 4; + const float* max_s_ptr = std::max_element(scores_ptr, scores_ptr + num_labels); + float score = *max_s_ptr; + if (score > confidence_threshold) + { + float x = *bboxes_ptr++; + float y = *bboxes_ptr++; + float w = *bboxes_ptr++; + float h = *bboxes_ptr; + + float x0 = clampf((x - 0.5f * w), 0.f, (float)infer_img_width); + float y0 = clampf((y - 0.5f * h), 0.f, (float)infer_img_height); + float x1 = clampf((x + 0.5f * w), 0.f, (float)infer_img_width); + float y1 = clampf((y + 0.5f * h), 0.f, (float)infer_img_height); + + cv::Rect_ bbox; + bbox.x = x0; + bbox.y = y0; + bbox.width = x1 - x0; + bbox.height = y1 - y0; + Object object; + object.label = max_s_ptr - scores_ptr; + object.prob = score; + object.rect = bbox; + detections.push_back(object); + } + } + objects = detections; +} + + +int AiEngineInferencevNcnn::detect_yolov8(const cv::Mat& bgr, std::vector& objects) +{ + /* + int img_w = bgr.cols; + int img_h = bgr.rows; + + // letterbox pad to multiple of MAX_STRIDE + int w = img_w; + int h = img_h; + float scale = 1.f; + if (w > h) + { + scale = (float)target_size / w; + w = target_size; + h = h * scale; + } + else + { + scale = (float)target_size / h; + h = target_size; + w = w * scale; + } + */ + int w = 640; + int h = 640; + int img_w = 640; + int img_h = 640; + float scale = 1.f; + + + ncnn::Mat in = ncnn::Mat::from_pixels_resize(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, img_w, img_h, w, h); + + int wpad = (target_size + MAX_STRIDE - 1) / MAX_STRIDE * MAX_STRIDE - w; + int hpad = (target_size + MAX_STRIDE - 1) / MAX_STRIDE * MAX_STRIDE - h; + ncnn::Mat in_pad; + ncnn::copy_make_border(in, in_pad, hpad / 2, hpad - hpad / 2, wpad / 2, wpad - wpad / 2, ncnn::BORDER_CONSTANT, 114.f); + + const float norm_vals[3] = {1 / 255.f, 1 / 255.f, 1 / 255.f}; + in_pad.substract_mean_normalize(0, norm_vals); + + auto start = std::chrono::high_resolution_clock::now(); + + ncnn::Extractor ex = yolov8.create_extractor(); + + ex.input("in0", in_pad); + + std::vector proposals; + + // stride 32 + { + ncnn::Mat out; + ex.extract("out0", out); + + std::vector objects32; + parse_yolov8_detections( + (float*)out.data, prob_threshold, + out.h, out.w, num_labels, + in_pad.w, in_pad.h, + objects32); + proposals.insert(proposals.end(), objects32.begin(), objects32.end()); + } + + // sort all proposals by score from highest to lowest + qsort_descent_inplace(proposals); + + // apply nms with nms_threshold + std::vector picked; + nms_sorted_bboxes(proposals, picked, nms_threshold); + + int count = picked.size(); + + objects.resize(count); + for (int i = 0; i < count; i++) + { + objects[i] = proposals[picked[i]]; + + // adjust offset to original unpadded + float x0 = (objects[i].rect.x - (wpad / 2)) / scale; + float y0 = (objects[i].rect.y - (hpad / 2)) / scale; + float x1 = (objects[i].rect.x + objects[i].rect.width - (wpad / 2)) / scale; + float y1 = (objects[i].rect.y + objects[i].rect.height - (hpad / 2)) / scale; + + // clip + x0 = std::max(std::min(x0, (float)(img_w - 1)), 0.f); + y0 = std::max(std::min(y0, (float)(img_h - 1)), 0.f); + x1 = std::max(std::min(x1, (float)(img_w - 1)), 0.f); + y1 = std::max(std::min(y1, (float)(img_h - 1)), 0.f); + + objects[i].rect.x = x0; + objects[i].rect.y = y0; + objects[i].rect.width = x1 - x0; + objects[i].rect.height = y1 - y0; + } + + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(end - start); + std::cout << "Time taken: " << duration.count() << " milliseconds" << std::endl; + return 0; +} + + +static cv::Mat draw_objects(const cv::Mat& bgr, const std::vector& objects) +{ + static const char* class_names[] = { + "Armoured vehicles", + "Truck", + "Passenger car", + "Artillery", + "Shadow of the vehicle", + "Trenches", + "Military", + "Ramps", + "Tank with additional protection", + "Smoke" + }; + + static const unsigned char colors[19][3] = { + {54, 67, 244}, + {99, 30, 233}, + {176, 39, 156}, + {183, 58, 103}, + {181, 81, 63}, + {243, 150, 33}, + {244, 169, 3}, + {212, 188, 0}, + {136, 150, 0}, + {80, 175, 76}, + {74, 195, 139}, + {57, 220, 205}, + {59, 235, 255}, + {7, 193, 255}, + {0, 152, 255}, + {34, 87, 255}, + {72, 85, 121}, + {158, 158, 158}, + {139, 125, 96} + }; + + int color_index = 0; + + cv::Mat image = bgr.clone(); + + for (size_t i = 0; i < objects.size(); i++) + { + const Object& obj = objects[i]; + + const unsigned char* color = colors[color_index % 19]; + color_index++; + + cv::Scalar cc(color[0], color[1], color[2]); + + fprintf(stderr, "%d = %.5f at %.2f %.2f %.2f x %.2f\n", obj.label, obj.prob, + obj.rect.x, obj.rect.y, obj.rect.width, obj.rect.height); + + cv::rectangle(image, obj.rect, cc, 2); + + char text[256]; + sprintf(text, "%s %.1f%%", class_names[obj.label], obj.prob * 100); + + int baseLine = 0; + cv::Size label_size = cv::getTextSize(text, cv::FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine); + + int x = obj.rect.x; + int y = obj.rect.y - label_size.height - baseLine; + if (y < 0) + y = 0; + if (x + label_size.width > image.cols) + x = image.cols - label_size.width; + + cv::rectangle(image, cv::Rect(cv::Point(x, y), cv::Size(label_size.width, label_size.height + baseLine)), + cc, -1); + + cv::putText(image, text, cv::Point(x, y + label_size.height), + cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(255, 255, 255)); + } + + return image; +} + + +void AiEngineInferencevNcnn::performInferenceSlot(cv::Mat frame) +{ + mActive = true; + + cv::Mat scaledImage = resizeAndPad(frame); + + std::vector objects; + detect_yolov8(scaledImage, objects); + + if (objects.empty() == false) { + AiEngineInferenceResult result; + result.frame = draw_objects(scaledImage, objects); + + for (uint i = 0; i < objects.size(); i++) { + const Object &detection = objects[i]; + AiEngineObject object; + object.classId = detection.label; + object.classStr = mClassNames[detection.label]; + object.propability = detection.prob; + object.rectangle.top = detection.rect.y; + object.rectangle.left = detection.rect.x; + object.rectangle.bottom = detection.rect.y + detection.rect.height; + object.rectangle.right = detection.rect.x + detection.rect.width; + result.objects.append(object); + } + + emit resultsReady(result); + } + + mActive = false; +} + + +void AiEngineInferencevNcnn::initialize(int number) +{ + (void)number; +} diff --git a/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.h b/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.h new file mode 100644 index 0000000..da0af35 --- /dev/null +++ b/misc/rtsp_ai_player/src-ncnn/aiengineinferencencnn.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include "aiengineinference.h" + + +struct Object +{ + cv::Rect_ rect; + int label; + float prob; +}; + + +class AiEngineInferencevNcnn : public AiEngineInference +{ + Q_OBJECT +public: + explicit AiEngineInferencevNcnn(QString modelPath, QObject *parent = nullptr); + void initialize(int number); + +public slots: + void performInferenceSlot(cv::Mat frame) override; + +private: + int detect_yolov8(const cv::Mat& bgr, std::vector& objects); + ncnn::Net yolov8; +}; + diff --git a/misc/rtsp_ai_player/src-onnx-runtime/aiengineinferenceonnxruntime.cpp b/misc/rtsp_ai_player/src-onnx-runtime/aiengineinferenceonnxruntime.cpp index ab74d32..2242bd6 100644 --- a/misc/rtsp_ai_player/src-onnx-runtime/aiengineinferenceonnxruntime.cpp +++ b/misc/rtsp_ai_player/src-onnx-runtime/aiengineinferenceonnxruntime.cpp @@ -4,9 +4,9 @@ #include "aiengineinferenceonnxruntime.h" -static const float confThreshold = 0.2f; -static const float iouThreshold = 0.4f; -static const float maskThreshold = 0.5f; +static const float confThreshold = 0.25f; +static const float iouThreshold = 0.45f; +static const float maskThreshold = 0.45f; AiEngineInferencevOnnxRuntime::AiEngineInferencevOnnxRuntime(QString modelPath, QObject *parent) : diff --git a/misc/rtsp_ai_player/src-opi5/aiengineinferenceopi5.cpp b/misc/rtsp_ai_player/src-opi5/aiengineinferenceopi5.cpp index 94f3b7f..54421ab 100644 --- a/misc/rtsp_ai_player/src-opi5/aiengineinferenceopi5.cpp +++ b/misc/rtsp_ai_player/src-opi5/aiengineinferenceopi5.cpp @@ -70,6 +70,7 @@ void AiEngineInferenceOpi5::freeImageBuffer(image_buffer_t& imgBuffer) cv::Mat AiEngineInferenceOpi5::resizeToHalfAndAssigntoTopLeft640x640(const cv::Mat& inputFrame) { + /* // Resize input frame to half size cv::Mat resizedFrame; cv::resize(inputFrame, resizedFrame, cv::Size(), 0.5, 0.5); @@ -81,6 +82,25 @@ cv::Mat AiEngineInferenceOpi5::resizeToHalfAndAssigntoTopLeft640x640(const cv::M cv::Rect roi(0, 0, resizedFrame.cols, resizedFrame.rows); resizedFrame.copyTo(outputFrame(roi)); + return outputFrame; + */ + + const int targetWidth = 640; + const int targetHeight = 640; + float aspectRatio = static_cast(inputFrame.cols) / static_cast(inputFrame.rows); + int newWidth = targetWidth; + int newHeight = static_cast(targetWidth / aspectRatio); + if (newHeight > targetHeight) { + newHeight = targetHeight; + newWidth = static_cast(targetHeight * aspectRatio); + } + + cv::Mat resizedFrame; + cv::resize(inputFrame, resizedFrame, cv::Size(newWidth, newHeight)); + cv::Mat outputFrame = cv::Mat::zeros(targetHeight, targetWidth, inputFrame.type()); + cv::Rect roi(cv::Point(0, 0), resizedFrame.size()); + resizedFrame.copyTo(outputFrame(roi)); + return outputFrame; } @@ -91,7 +111,9 @@ void AiEngineInferenceOpi5::drawObjects(cv::Mat& image, const object_detect_resu const object_detect_result& result = result_list.results[i]; if (result.cls_id >= mClassNames.size()) { - continue; + //result.cls_id = result.cls_id % mClassNames.size(); + qDebug() << "Class id >= mClassNames.size() Reducing it."; + //continue; } fprintf(stderr, "TUOMAS [%d] prop = %f\n", i, result.prop); @@ -106,7 +128,7 @@ void AiEngineInferenceOpi5::drawObjects(cv::Mat& image, const object_detect_resu // Text char c_text[256]; //sprintf(c_text, "%s %d%%", coco_cls_to_name(result.cls_id), (int)(round(result.prop * 100))); - sprintf(c_text, "%s %d%%", mClassNames[result.cls_id].toStdString().c_str(), (int)(round(result.prop * 100))); + sprintf(c_text, "%s %d%%", mClassNames[result.cls_id % mClassNames.size()].toStdString().c_str(), (int)(round(result.prop * 100))); cv::Point textOrg(left, top - 5); cv::putText(image, std::string(c_text), textOrg, cv::FONT_HERSHEY_COMPLEX, result.prop, cv::Scalar(0, 0, 255), 1, cv::LINE_AA); } @@ -131,23 +153,25 @@ void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame) return; } - AiEngineInferenceResult result; - for (int i = 0; i < od_results.count; i++) { - object_detect_result *det_result = &(od_results.results[i]); + if (od_results.count > 0) { + AiEngineInferenceResult result; + for (int i = 0; i < od_results.count; i++) { + object_detect_result *det_result = &(od_results.results[i]); + qDebug() << "TUOMAS box:" << det_result->box.top << det_result->box.left << det_result->box.bottom << det_result->box.right; + AiEngineObject object; + object.classId = det_result->cls_id; + object.propability = det_result->prop; + object.rectangle.top = det_result->box.top; + object.rectangle.left = det_result->box.left; + object.rectangle.bottom = det_result->box.bottom; + object.rectangle.right = det_result->box.right; + result.objects.append(object); + } - AiEngineObject object; - object.classId = det_result->cls_id; - object.propability = det_result->prop; - object.rectangle.top = det_result->box.top; - object.rectangle.left = det_result->box.left; - object.rectangle.bottom = det_result->box.bottom; - object.rectangle.right = det_result->box.right; - result.objects.append(object); + drawObjects(scaledFrame, od_results); + result.frame = scaledFrame.clone(); + emit resultsReady(result); } - drawObjects(scaledFrame, od_results); - result.frame = scaledFrame.clone(); - emit resultsReady(result); - mActive = false; }