Addd initial AiEngineGimbalControl class

tmp/opi_rtsp/aiengine.cpp

@@ -26,6 +26,8 @@ AiEngine::AiEngine(QString modelPath, QObject *parent)
     connect(mInference, &AiEngineInference::resultsReady, this, &AiEngine::inferenceResultsReceivedSlot, Qt::QueuedConnection);
     connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection);
     inferenceThread->start();
+
+    mGimbalControl =  new AiEngineGimbalControl(this);
 }
 
 
@@ -41,15 +43,15 @@ void AiEngine::stop(void)
 }
 
 
-void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult results)
+void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult result)
 {
-    (void)results;
+    //qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId();
-    qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId();
 
-    cv::imshow("Received Frame", results.frame);
+    mGimbalControl->inferenceResultSlot(result);
+    cv::imshow("Received Frame", result.frame);
 
 //#ifndef OPI5_BUILD
-    //cv::imshow("Received Frame", results.frame);
+    //cv::imshow("Received Frame", result.frame);
 //#endif
 }
 
@@ -60,7 +62,7 @@ void AiEngine::frameReceivedSlot(cv::Mat frame)
     //cv::imshow("Received Frame", frame);
 
     if (mInference->isActive() == false) {
-        qDebug() << "AiEngine. Inference thread is free. Sending frame to it.";
+        //qDebug() << "AiEngine. Inference thread is free. Sending frame to it.";
         emit inferenceFrame(frame.clone());
     }
 }

tmp/opi_rtsp/aiengine.h

@@ -5,6 +5,7 @@
 #include <opencv2/videoio.hpp>
 #include "aienginertsplistener.h"
 #include "aiengineinference.h"
+#include "aienginegimbalcontrol.h"
 
 class AiEngine : public QObject
 {
@@ -16,7 +17,7 @@ public:
 
 public slots:
     void frameReceivedSlot(cv::Mat frame);
-    void inferenceResultsReceivedSlot(AiEngineInferenceResult results);
+    void inferenceResultsReceivedSlot(AiEngineInferenceResult result);
 
 signals:
     void inferenceFrame(cv::Mat frame);
@@ -24,6 +25,5 @@ signals:
 private:
     AiEngineRtspListener *mRtspListener;
     AiEngineInference *mInference;
-
+    AiEngineGimbalControl *mGimbalControl;
-signals:
 };

tmp/opi_rtsp/aienginegimbalcontrol.cpp

@@ -0,0 +1,49 @@
+#include "aienginegimbalcontrol.h"
+
+
+AiEngineGimbalControl::AiEngineGimbalControl(QObject *parent)
+    : QObject{parent}
+{}
+
+
+AiEngineRectangle AiEngineGimbalControl::getGroupCoordinates(QVector<AiEngineObject> &objects)
+{
+    AiEngineRectangle groupRectangle;
+    groupRectangle.top    = 1000000;
+    groupRectangle.left   = 1000000;
+    groupRectangle.bottom = 0;
+    groupRectangle.right  = 0;
+
+    for (int i = 0; i < objects.size(); i++) {
+        const AiEngineRectangle &objectRectangle = objects[i].rectangle;
+
+        if (objectRectangle.top < groupRectangle.top) {
+            groupRectangle.top = objectRectangle.top;
+        }
+        if (objectRectangle.left < groupRectangle.left) {
+            groupRectangle.left = objectRectangle.left;
+        }
+        if (objectRectangle.bottom > groupRectangle.bottom) {
+            groupRectangle.bottom = objectRectangle.bottom;
+        }
+        if (objectRectangle.right > groupRectangle.right) {
+            groupRectangle.right = objectRectangle.right;
+        }
+    }
+
+    return groupRectangle;
+}
+
+
+void AiEngineGimbalControl::inferenceResultSlot(AiEngineInferenceResult result)
+{
+    // We got list of all recognized objects, but at least for now we will zoom to all objects at
+    // once and not for each invidually. Got minimal coordinates which contains the all objects.
+    AiEngineRectangle groupRect = getGroupCoordinates(result.objects);
+
+    // AI did inference with 640x360 resolution. Scale back to A8's 1280x720 resolution.
+    groupRect.top    *= 2;
+    groupRect.left   *= 2;
+    groupRect.bottom *= 2;
+    groupRect.right  *= 2;
+}

tmp/opi_rtsp/aienginegimbalcontrol.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <QObject>
+#include <QElapsedTimer>
+#include "aiengineinference.h"
+
+class AiEngineGimbalControl : public QObject
+{
+    Q_OBJECT
+public:
+    explicit AiEngineGimbalControl(QObject *parent = nullptr);
+
+private:
+    AiEngineRectangle getGroupCoordinates(QVector<AiEngineObject> &objects);
+
+public slots:
+    void inferenceResultSlot(AiEngineInferenceResult results);
+};

tmp/opi_rtsp/aiengineinference.cpp

@@ -1,5 +1,6 @@
 #include "aiengineinference.h"
 
+
 AiEngineInference::AiEngineInference(QString modelPath, QObject *parent)
     : QObject{parent},
       mModelPath(modelPath),

tmp/opi_rtsp/aiengineinference.h

@@ -2,13 +2,31 @@
 
 #include <QObject>
 #include <QString>
+#include <QVector>
 #include <opencv2/core.hpp>
 
 
+class AiEngineRectangle {
+public:
+    int left;
+    int top;
+    int right;
+    int bottom;
+};
+
+
+class AiEngineObject {
+public:
+    AiEngineRectangle rectangle;
+    float propability;
+    int classId;
+};
+
+
 class AiEngineInferenceResult {
 public:
     cv::Mat frame;
-    int objects;
+    QVector<AiEngineObject> objects;
 };
 
 

tmp/opi_rtsp/opi_rtsp.pro

@@ -1,11 +1,11 @@
 QT += core
 CONFIG += c++11 link_pkgconfig concurrent
 PKGCONFIG += opencv4
-
-SOURCES += $$PWD/*.cpp
-HEADERS += $$PWD/*.h
-
 MOC_DIR = moc
+OBJECTS_DIR = obj
+
+SOURCES = $$PWD/*.cpp
+HEADERS = $$PWD/*.h
 
 opi5 {
     message("OPI5 build")
@@ -13,7 +13,6 @@ opi5 {
     INCLUDEPATH += /usr/include/rga # not correct in pkg-config file
     QMAKE_CXXFLAGS += -DOPI5_BUILD
     LIBS += /usr/local/lib/librknnrt.so
-    OBJECTS_DIR = objs-opi5
     SOURCES += $$PWD/src-opi5/*.c $$PWD/src-opi5/*.cpp $$PWD/src-opi5/*.cc
     HEADERS += $$PWD/src-opi5/*.h
 } else {
@@ -23,7 +22,6 @@ opi5 {
     INCLUDEPATH += /opt/onnxruntime-linux-x64-1.18.0/include
     LIBS += /opt/onnxruntime-linux-x64-1.18.0/lib/libonnxruntime.so.1.18.0
     QMAKE_LFLAGS += -Wl,-rpath,/opt/onnxruntime-linux-x64-1.18.0/lib
-    OBJECTS_DIR = objs-onnx
     SOURCES += $$PWD/src-onnx/*.cpp
     HEADERS += $$PWD/src-onnx/*.h
 }

tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp

@@ -4,6 +4,9 @@
 #include <opencv2/highgui.hpp>
 #include <iomanip>
 #include "aiengineinferenceopi5.h"
+#include "file_utils.h"
+#include "image_drawing.h"
+
 
 AiEngineInferenceOpi5::AiEngineInferenceOpi5(QString modelPath, QObject *parent)
     : AiEngineInference{modelPath, parent}
@@ -28,7 +31,7 @@ AiEngineInferenceOpi5::~AiEngineInferenceOpi5()
 }
 
 
-image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame)
+image_buffer_t AiEngineInferenceOpi5::convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame)
 {
     // Convert BGR to RGB
     cv::Mat rgbFrame;
@@ -50,7 +53,7 @@ image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame)
 }
 
 
-void freeImageBuffer(image_buffer_t& imgBuffer)
+void AiEngineInferenceOpi5::freeImageBuffer(image_buffer_t& imgBuffer)
 {
     if (imgBuffer.virt_addr) {
         delete[] imgBuffer.virt_addr;
@@ -59,42 +62,25 @@ void freeImageBuffer(image_buffer_t& imgBuffer)
 }
 
 
-cv::Mat resizeAndCenterImage(const cv::Mat& src, int targetWidth, int targetHeight)
+cv::Mat AiEngineInferenceOpi5::resizeToHalfAndAssigntoTopLeft640x640(const cv::Mat& inputFrame)
 {
-    // Calculate the scaling factor and size
+    // Resize input frame to half size
-    int originalWidth = src.cols;
+    cv::Mat resizedFrame;
-    int originalHeight = src.rows;
+    cv::resize(inputFrame, resizedFrame, cv::Size(), 0.5, 0.5);
-    double aspectRatio = static_cast<double>(originalWidth) / originalHeight;
-    int newWidth, newHeight;
 
-    if (originalWidth > originalHeight) {
+    // Create a 640x640 frame to place the resized frame
-        newWidth = targetWidth;
+    cv::Mat outputFrame = cv::Mat::zeros(640, 640, inputFrame.type());
-        newHeight = static_cast<int>(targetWidth / aspectRatio);
-    }
-    else {
-        newHeight = targetHeight;
-        newWidth = static_cast<int>(targetHeight * aspectRatio);
-    }
 
-    // Resize the image
+    // Copy the resized frame to the top-left corner of the output frame
-    cv::Mat resizedImage;
+    cv::Rect roi(0, 0, resizedFrame.cols, resizedFrame.rows);
-    cv::resize(src, resizedImage, cv::Size(newWidth, newHeight));
+    resizedFrame.copyTo(outputFrame(roi));
 
-    // Create a black image of target size
+    return outputFrame;
-    cv::Mat outputImage = cv::Mat::zeros(targetHeight, targetWidth, src.type());
-
-    // Calculate position to center the image
-    int top = (targetHeight - newHeight) / 2;
-    int left = (targetWidth - newWidth) / 2;
-
-    // Copy resized image into the center of the black image
-    resizedImage.copyTo(outputImage(cv::Rect(left, top, newWidth, newHeight)));
-
-    return outputImage;
 }
 
 
-void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) {
+void AiEngineInferenceOpi5::drawObjects(cv::Mat& image, const object_detect_result_list& result_list)
+{
     for (int i = 0; i < result_list.count; i++) {
         const object_detect_result& result = result_list.results[i];
 
@@ -105,7 +91,6 @@ void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) {
 
         cv::rectangle(image, cv::Point(left, top), cv::Point(right, bottom), cv::Scalar(255, 0, 0), 2);
 
-        fprintf(stderr, "TUOMAS OMA. ID=%d  String=%s\n", result.cls_id, coco_cls_to_name(result.cls_id));
         // Text
         char c_text[256];
         sprintf(c_text, "%s %.1f%%", coco_cls_to_name(result.cls_id), result.prop * 100);
@@ -117,22 +102,32 @@ void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) {
 
 void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame)
 {
-    qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId();
+    //qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId();
 
     mActive = true;
 
-    cv::Mat scaledFrame      = resizeAndCenterImage(frame, 640, 640);
+    cv::Mat scaledFrame      = resizeToHalfAndAssigntoTopLeft640x640(frame);
     image_buffer_t imgBuffer = convertCV2FrameToImageBuffer(scaledFrame);
 
     object_detect_result_list od_results;
     int ret = inference_yolov8_model(&mRrknnAppCtx, &imgBuffer, &od_results);
     if (ret != 0) {
         qDebug() << "inference_yolov8_model() failure! ret: " << ret;
+        return;
     }
 
+    AiEngineInferenceResult result;
     for (int i = 0; i < od_results.count; i++) {
         object_detect_result *det_result = &(od_results.results[i]);
-        fprintf(stderr, "TUOMAS RKNN. ID=%d  String=%s\n", det_result->cls_id, coco_cls_to_name(det_result->cls_id));
+
+        AiEngineObject object;
+        object.classId          = det_result->cls_id;
+        object.propability      = det_result->prop;
+        object.rectangle.top    = det_result->box.top;
+        object.rectangle.left   = det_result->box.left;
+        object.rectangle.bottom = det_result->box.bottom;
+        object.rectangle.right  = det_result->box.right;
+        result.objects.append(object);
     }
 
     /*
@@ -154,13 +149,6 @@ void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame)
     }
     */
 
-    // Testing
-
-    //write_image(filename.c_str(), &imgBuffer);
-
-    drawObjects(scaledFrame, od_results);
-    freeImageBuffer(imgBuffer);
-
     /*
     static int imageNum = 0;
     std::stringstream ss;
@@ -170,9 +158,10 @@ void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame)
     cv::imwrite(filename, scaledFrame);
     */
 
-    AiEngineInferenceResult result;
+    drawObjects(scaledFrame, od_results);
+    freeImageBuffer(imgBuffer);
+
     result.frame = scaledFrame.clone();
-    result.objects = od_results.count;
     emit resultsReady(result);
 
     mActive = false;

tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h

@@ -4,8 +4,6 @@
 #include "aiengineinference.h"
 #include "yolov8.h"
 #include "image_utils.h"
-#include "file_utils.h"
-#include "image_drawing.h"
 
 class AiEngineInferenceOpi5 : public AiEngineInference
 {
@@ -18,5 +16,10 @@ public slots:
     void performInferenceSlot(cv::Mat frame) override;
 
 private:
+    image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame);
+    void freeImageBuffer(image_buffer_t& imgBuffer);
+    cv::Mat resizeToHalfAndAssigntoTopLeft640x640(const cv::Mat& inputFrame);
+    void drawObjects(cv::Mat& image, const object_detect_result_list& result_list);
+
     rknn_app_context_t mRrknnAppCtx;
 };