Fully working OPI5 and Azaion AI object recognition

tmp/opi_rtsp/aiengine.cpp

@@ -2,8 +2,12 @@
 #include <opencv2/highgui.hpp>
 #include "aiengine.h"
 #include "aiengineinference.h"
-#include "src-onnx/aiengineinferenceonnx.h"
 
+#ifdef OPI5_BUILD
+#include "src-opi5/aiengineinferenceopi5.h"
+#else
+#include "src-onnx/aiengineinferenceonnx.h"
+#endif
 
 AiEngine::AiEngine(QString modelPath, QObject *parent)
     : QObject{parent}
@@ -11,8 +15,13 @@ AiEngine::AiEngine(QString modelPath, QObject *parent)
     mRtspListener = new AiEngineRtspListener(this);
     connect(mRtspListener, &AiEngineRtspListener::frameReceived, this, &AiEngine::frameReceivedSlot);
 
+#ifdef OPI5_BUILD
+    mInference = new AiEngineInferenceOpi5(modelPath);
+#else
+    mInference = new AiEngineInferenceOnnx(modelPath);
+#endif
+
     QThread *inferenceThread = new QThread(this);
-    mInference               = new AiEngineInferenceOnnx(modelPath);
     mInference->moveToThread(inferenceThread);
     connect(mInference, &AiEngineInference::resultsReady, this, &AiEngine::inferenceResultsReceivedSlot, Qt::QueuedConnection);
     connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection);
@@ -36,7 +45,12 @@ void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult results)
 {
     (void)results;
     qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId();
+
     cv::imshow("Received Frame", results.frame);
+
+//#ifndef OPI5_BUILD
+    //cv::imshow("Received Frame", results.frame);
+//#endif
 }
 
 

tmp/opi_rtsp/azaion_10_labels_list.txt

@@ -0,0 +1,10 @@
+Armoured_vehicle
+Truck
+Vehicle
+Artillery
+Shadow_artillery
+Tranches
+Military man
+Tyre_tracks
+Additional_protection_tank
+Smoke

tmp/opi_rtsp/config.h

@@ -2,4 +2,9 @@
 
 #include <QString>
 
+#ifdef OPI5_BUILD
+//QString rtspVideoUrl = "rtsp://192.168.168.11:8554/live.stream";
+QString rtspVideoUrl = "rtsp://192.168.0.25:8554/main.264";
+#else
 QString rtspVideoUrl = "rtsp://localhost:8554/live.stream";
+#endif

tmp/opi_rtsp/main.cpp

@@ -20,8 +20,6 @@ int main(int argc, char *argv[])
         return 1;
     }
 
-    cv::namedWindow("AiEngine", cv::WINDOW_AUTOSIZE);
-
     AiEngine aiEngine(modelFile);
     aiEngine.start();
 

tmp/opi_rtsp/opi_rtsp.pro

@@ -2,8 +2,10 @@ QT += core
 CONFIG += c++11 link_pkgconfig concurrent
 PKGCONFIG += opencv4
 
-SOURCES += *.cpp
+SOURCES += $$PWD/*.cpp
-HEADERS += *.h
+HEADERS += $$PWD/*.h
+
+MOC_DIR = moc
 
 opi5 {
     message("OPI5 build")
@@ -12,10 +14,9 @@ opi5 {
     QMAKE_CXXFLAGS += -DOPI5_BUILD
     LIBS += /usr/local/lib/librknnrt.so
     OBJECTS_DIR = objs-opi5
-    SOURCES += src-opi5/*.c src-opi5/*.cc
+    SOURCES += $$PWD/src-opi5/*.c $$PWD/src-opi5/*.cpp $$PWD/src-opi5/*.cc
-    HEADERS += src-opi5/*.h
+    HEADERS += $$PWD/src-opi5/*.h
-}
+} else {
-else {
     message("ONNX build")
     message("You must use YOLOv10 ONNX files")
     QMAKE_CXXFLAGS += -DONNX_BUILD
@@ -23,6 +24,6 @@ else {
     LIBS += /opt/onnxruntime-linux-x64-1.18.0/lib/libonnxruntime.so.1.18.0
     QMAKE_LFLAGS += -Wl,-rpath,/opt/onnxruntime-linux-x64-1.18.0/lib
     OBJECTS_DIR = objs-onnx
-    SOURCES += src-onnx/*.cpp
+    SOURCES += $$PWD/src-onnx/*.cpp
-    HEADERS += src-onnx/*.h
+    HEADERS += $$PWD/src-onnx/*.h
 }

tmp/opi_rtsp/src-onnx/aiengineinferenceonnx.cpp

@@ -1,7 +1,5 @@
-#include <QThread>
 #include <QDebug>
-#include <thread>
+#include <QThread>
-#include <chrono>
 #include "aiengineinferenceonnx.h"
 
 AiEngineInferenceOnnx::AiEngineInferenceOnnx(QString modelPath, QObject *parent)

tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp

@@ -0,0 +1,179 @@
+#include <QDebug>
+#include <QThread>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <iomanip>
+#include "aiengineinferenceopi5.h"
+
+AiEngineInferenceOpi5::AiEngineInferenceOpi5(QString modelPath, QObject *parent)
+    : AiEngineInference{modelPath, parent}
+{
+    qDebug() << "AiEngineInferenceOpi5() test mModelPath=" << mModelPath;
+
+    memset(&mRrknnAppCtx, 0, sizeof(rknn_app_context_t));
+    init_post_process();
+
+    int ret = init_yolov8_model(modelPath.toLocal8Bit(), &mRrknnAppCtx);
+    if (ret != 0) {
+        qDebug() << "init_yolov8_model() failure! ret: " << ret << "modelPath = " << modelPath;
+        return;
+    }
+}
+
+
+AiEngineInferenceOpi5::~AiEngineInferenceOpi5()
+{
+    deinit_post_process();
+    release_yolov8_model(&mRrknnAppCtx);
+}
+
+
+image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame)
+{
+    // Convert BGR to RGB
+    cv::Mat rgbFrame;
+    cv::cvtColor(bgrFrame, rgbFrame, cv::COLOR_BGR2RGB);
+
+    image_buffer_t imgBuffer;
+    memset(&imgBuffer, 0, sizeof(image_buffer_t));
+    imgBuffer.width         = rgbFrame.cols;
+    imgBuffer.height        = rgbFrame.rows;
+    imgBuffer.width_stride  = rgbFrame.step;
+    imgBuffer.height_stride = rgbFrame.rows;
+    imgBuffer.format        = IMAGE_FORMAT_RGB888;
+    imgBuffer.size          = rgbFrame.total() * rgbFrame.elemSize();
+
+    imgBuffer.virt_addr = new unsigned char[imgBuffer.size];
+    std::memcpy(imgBuffer.virt_addr, rgbFrame.data, imgBuffer.size);
+
+    return imgBuffer;
+}
+
+
+void freeImageBuffer(image_buffer_t& imgBuffer)
+{
+    if (imgBuffer.virt_addr) {
+        delete[] imgBuffer.virt_addr;
+        imgBuffer.virt_addr = nullptr;
+    }
+}
+
+
+cv::Mat resizeAndCenterImage(const cv::Mat& src, int targetWidth, int targetHeight)
+{
+    // Calculate the scaling factor and size
+    int originalWidth = src.cols;
+    int originalHeight = src.rows;
+    double aspectRatio = static_cast<double>(originalWidth) / originalHeight;
+    int newWidth, newHeight;
+
+    if (originalWidth > originalHeight) {
+        newWidth = targetWidth;
+        newHeight = static_cast<int>(targetWidth / aspectRatio);
+    }
+    else {
+        newHeight = targetHeight;
+        newWidth = static_cast<int>(targetHeight * aspectRatio);
+    }
+
+    // Resize the image
+    cv::Mat resizedImage;
+    cv::resize(src, resizedImage, cv::Size(newWidth, newHeight));
+
+    // Create a black image of target size
+    cv::Mat outputImage = cv::Mat::zeros(targetHeight, targetWidth, src.type());
+
+    // Calculate position to center the image
+    int top = (targetHeight - newHeight) / 2;
+    int left = (targetWidth - newWidth) / 2;
+
+    // Copy resized image into the center of the black image
+    resizedImage.copyTo(outputImage(cv::Rect(left, top, newWidth, newHeight)));
+
+    return outputImage;
+}
+
+
+void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) {
+    for (int i = 0; i < result_list.count; i++) {
+        const object_detect_result& result = result_list.results[i];
+
+        int left = result.box.left;
+        int top = result.box.top;
+        int right = result.box.right;
+        int bottom = result.box.bottom;
+
+        cv::rectangle(image, cv::Point(left, top), cv::Point(right, bottom), cv::Scalar(255, 0, 0), 2);
+
+        fprintf(stderr, "TUOMAS OMA. ID=%d  String=%s\n", result.cls_id, coco_cls_to_name(result.cls_id));
+        // Text
+        char c_text[256];
+        sprintf(c_text, "%s %.1f%%", coco_cls_to_name(result.cls_id), result.prop * 100);
+        cv::Point textOrg(left, top - 5);
+        cv::putText(image, std::string(c_text), textOrg, cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 255), 1);
+    }
+}
+
+
+void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame)
+{
+    qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId();
+
+    mActive = true;
+
+    cv::Mat scaledFrame      = resizeAndCenterImage(frame, 640, 640);
+    image_buffer_t imgBuffer = convertCV2FrameToImageBuffer(scaledFrame);
+
+    object_detect_result_list od_results;
+    int ret = inference_yolov8_model(&mRrknnAppCtx, &imgBuffer, &od_results);
+    if (ret != 0) {
+        qDebug() << "inference_yolov8_model() failure! ret: " << ret;
+    }
+
+    for (int i = 0; i < od_results.count; i++) {
+        object_detect_result *det_result = &(od_results.results[i]);
+        fprintf(stderr, "TUOMAS RKNN. ID=%d  String=%s\n", det_result->cls_id, coco_cls_to_name(det_result->cls_id));
+    }
+
+    /*
+    char text[256];
+    for (int i = 0; i < od_results.count; i++) {
+        object_detect_result *det_result = &(od_results.results[i]);
+        printf("%s @ (%d %d %d %d) %.3f\n", coco_cls_to_name(det_result->cls_id),
+               det_result->box.left, det_result->box.top,
+               det_result->box.right, det_result->box.bottom,
+               det_result->prop);
+        int x1 = det_result->box.left;
+        int y1 = det_result->box.top;
+        int x2 = det_result->box.right;
+        int y2 = det_result->box.bottom;
+
+        draw_rectangle(&imgBuffer, x1, y1, x2 - x1, y2 - y1, COLOR_BLUE, 3);
+        sprintf(text, "%s %.1f%%", coco_cls_to_name(det_result->cls_id), det_result->prop * 100);
+        draw_text(&imgBuffer, text, x1, y1 - 20, COLOR_RED, 10);
+    }
+    */
+
+    // Testing
+
+    //write_image(filename.c_str(), &imgBuffer);
+
+    drawObjects(scaledFrame, od_results);
+    freeImageBuffer(imgBuffer);
+
+    /*
+    static int imageNum = 0;
+    std::stringstream ss;
+    ss << std::setw(4) << std::setfill('0') << imageNum++;
+    std::string formatted_number = ss.str();
+    std::string filename = "/tmp/out-" + formatted_number + ".png";
+    cv::imwrite(filename, scaledFrame);
+    */
+
+    AiEngineInferenceResult result;
+    result.frame = scaledFrame.clone();
+    result.objects = od_results.count;
+    emit resultsReady(result);
+
+    mActive = false;
+}

tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <QObject>
+#include "aiengineinference.h"
+#include "yolov8.h"
+#include "image_utils.h"
+#include "file_utils.h"
+#include "image_drawing.h"
+
+class AiEngineInferenceOpi5 : public AiEngineInference
+{
+    Q_OBJECT
+public:
+    explicit AiEngineInferenceOpi5(QString modelPath, QObject *parent = nullptr);
+    ~AiEngineInferenceOpi5();
+
+public slots:
+    void performInferenceSlot(cv::Mat frame) override;
+
+private:
+    rknn_app_context_t mRrknnAppCtx;
+};

tmp/opi_rtsp/src-opi5/postprocess.cc

@@ -23,7 +23,13 @@
 
 #include <set>
 #include <vector>
-#define LABEL_NALE_TXT_PATH "./model/coco_80_labels_list.txt"
+
+#ifdef OPI5_BUILD
+#define LABEL_NALE_TXT_PATH "azaion_10_labels_list.txt"
+#else
+#define LABEL_NALE_TXT_PATH "coco_80_labels_list.txt"
+#endif
+
 
 static char *labels[OBJ_CLASS_NUM];