diff --git a/tmp/opi_rtsp/aiengine.cpp b/tmp/opi_rtsp/aiengine.cpp index a4263f6..fd4df6f 100644 --- a/tmp/opi_rtsp/aiengine.cpp +++ b/tmp/opi_rtsp/aiengine.cpp @@ -26,6 +26,8 @@ AiEngine::AiEngine(QString modelPath, QObject *parent) connect(mInference, &AiEngineInference::resultsReady, this, &AiEngine::inferenceResultsReceivedSlot, Qt::QueuedConnection); connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection); inferenceThread->start(); + + mGimbalControl = new AiEngineGimbalControl(this); } @@ -41,15 +43,15 @@ void AiEngine::stop(void) } -void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult results) +void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult result) { - (void)results; - qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId(); + //qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId(); - cv::imshow("Received Frame", results.frame); + mGimbalControl->inferenceResultSlot(result); + cv::imshow("Received Frame", result.frame); //#ifndef OPI5_BUILD - //cv::imshow("Received Frame", results.frame); + //cv::imshow("Received Frame", result.frame); //#endif } @@ -60,7 +62,7 @@ void AiEngine::frameReceivedSlot(cv::Mat frame) //cv::imshow("Received Frame", frame); if (mInference->isActive() == false) { - qDebug() << "AiEngine. Inference thread is free. Sending frame to it."; + //qDebug() << "AiEngine. Inference thread is free. Sending frame to it."; emit inferenceFrame(frame.clone()); } } diff --git a/tmp/opi_rtsp/aiengine.h b/tmp/opi_rtsp/aiengine.h index d7f0ad1..8c9ddae 100644 --- a/tmp/opi_rtsp/aiengine.h +++ b/tmp/opi_rtsp/aiengine.h @@ -5,6 +5,7 @@ #include #include "aienginertsplistener.h" #include "aiengineinference.h" +#include "aienginegimbalcontrol.h" class AiEngine : public QObject { @@ -16,7 +17,7 @@ public: public slots: void frameReceivedSlot(cv::Mat frame); - void inferenceResultsReceivedSlot(AiEngineInferenceResult results); + void inferenceResultsReceivedSlot(AiEngineInferenceResult result); signals: void inferenceFrame(cv::Mat frame); @@ -24,6 +25,5 @@ signals: private: AiEngineRtspListener *mRtspListener; AiEngineInference *mInference; - -signals: + AiEngineGimbalControl *mGimbalControl; }; diff --git a/tmp/opi_rtsp/aienginegimbalcontrol.cpp b/tmp/opi_rtsp/aienginegimbalcontrol.cpp new file mode 100644 index 0000000..fb61564 --- /dev/null +++ b/tmp/opi_rtsp/aienginegimbalcontrol.cpp @@ -0,0 +1,49 @@ +#include "aienginegimbalcontrol.h" + + +AiEngineGimbalControl::AiEngineGimbalControl(QObject *parent) + : QObject{parent} +{} + + +AiEngineRectangle AiEngineGimbalControl::getGroupCoordinates(QVector &objects) +{ + AiEngineRectangle groupRectangle; + groupRectangle.top = 1000000; + groupRectangle.left = 1000000; + groupRectangle.bottom = 0; + groupRectangle.right = 0; + + for (int i = 0; i < objects.size(); i++) { + const AiEngineRectangle &objectRectangle = objects[i].rectangle; + + if (objectRectangle.top < groupRectangle.top) { + groupRectangle.top = objectRectangle.top; + } + if (objectRectangle.left < groupRectangle.left) { + groupRectangle.left = objectRectangle.left; + } + if (objectRectangle.bottom > groupRectangle.bottom) { + groupRectangle.bottom = objectRectangle.bottom; + } + if (objectRectangle.right > groupRectangle.right) { + groupRectangle.right = objectRectangle.right; + } + } + + return groupRectangle; +} + + +void AiEngineGimbalControl::inferenceResultSlot(AiEngineInferenceResult result) +{ + // We got list of all recognized objects, but at least for now we will zoom to all objects at + // once and not for each invidually. Got minimal coordinates which contains the all objects. + AiEngineRectangle groupRect = getGroupCoordinates(result.objects); + + // AI did inference with 640x360 resolution. Scale back to A8's 1280x720 resolution. + groupRect.top *= 2; + groupRect.left *= 2; + groupRect.bottom *= 2; + groupRect.right *= 2; +} diff --git a/tmp/opi_rtsp/aienginegimbalcontrol.h b/tmp/opi_rtsp/aienginegimbalcontrol.h new file mode 100644 index 0000000..63dc552 --- /dev/null +++ b/tmp/opi_rtsp/aienginegimbalcontrol.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include "aiengineinference.h" + +class AiEngineGimbalControl : public QObject +{ + Q_OBJECT +public: + explicit AiEngineGimbalControl(QObject *parent = nullptr); + +private: + AiEngineRectangle getGroupCoordinates(QVector &objects); + +public slots: + void inferenceResultSlot(AiEngineInferenceResult results); +}; diff --git a/tmp/opi_rtsp/aiengineinference.cpp b/tmp/opi_rtsp/aiengineinference.cpp index 356d1f4..cbb101f 100644 --- a/tmp/opi_rtsp/aiengineinference.cpp +++ b/tmp/opi_rtsp/aiengineinference.cpp @@ -1,5 +1,6 @@ #include "aiengineinference.h" + AiEngineInference::AiEngineInference(QString modelPath, QObject *parent) : QObject{parent}, mModelPath(modelPath), diff --git a/tmp/opi_rtsp/aiengineinference.h b/tmp/opi_rtsp/aiengineinference.h index 7fe0f2c..49f96e0 100644 --- a/tmp/opi_rtsp/aiengineinference.h +++ b/tmp/opi_rtsp/aiengineinference.h @@ -2,13 +2,31 @@ #include #include +#include #include +class AiEngineRectangle { +public: + int left; + int top; + int right; + int bottom; +}; + + +class AiEngineObject { +public: + AiEngineRectangle rectangle; + float propability; + int classId; +}; + + class AiEngineInferenceResult { public: cv::Mat frame; - int objects; + QVector objects; }; diff --git a/tmp/opi_rtsp/opi_rtsp.pro b/tmp/opi_rtsp/opi_rtsp.pro index b2e283c..a1a8c85 100644 --- a/tmp/opi_rtsp/opi_rtsp.pro +++ b/tmp/opi_rtsp/opi_rtsp.pro @@ -1,11 +1,11 @@ QT += core CONFIG += c++11 link_pkgconfig concurrent PKGCONFIG += opencv4 - -SOURCES += $$PWD/*.cpp -HEADERS += $$PWD/*.h - MOC_DIR = moc +OBJECTS_DIR = obj + +SOURCES = $$PWD/*.cpp +HEADERS = $$PWD/*.h opi5 { message("OPI5 build") @@ -13,7 +13,6 @@ opi5 { INCLUDEPATH += /usr/include/rga # not correct in pkg-config file QMAKE_CXXFLAGS += -DOPI5_BUILD LIBS += /usr/local/lib/librknnrt.so - OBJECTS_DIR = objs-opi5 SOURCES += $$PWD/src-opi5/*.c $$PWD/src-opi5/*.cpp $$PWD/src-opi5/*.cc HEADERS += $$PWD/src-opi5/*.h } else { @@ -23,7 +22,6 @@ opi5 { INCLUDEPATH += /opt/onnxruntime-linux-x64-1.18.0/include LIBS += /opt/onnxruntime-linux-x64-1.18.0/lib/libonnxruntime.so.1.18.0 QMAKE_LFLAGS += -Wl,-rpath,/opt/onnxruntime-linux-x64-1.18.0/lib - OBJECTS_DIR = objs-onnx SOURCES += $$PWD/src-onnx/*.cpp HEADERS += $$PWD/src-onnx/*.h } diff --git a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp index 0f6124d..dfcf411 100644 --- a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp +++ b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp @@ -4,6 +4,9 @@ #include #include #include "aiengineinferenceopi5.h" +#include "file_utils.h" +#include "image_drawing.h" + AiEngineInferenceOpi5::AiEngineInferenceOpi5(QString modelPath, QObject *parent) : AiEngineInference{modelPath, parent} @@ -28,7 +31,7 @@ AiEngineInferenceOpi5::~AiEngineInferenceOpi5() } -image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame) +image_buffer_t AiEngineInferenceOpi5::convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame) { // Convert BGR to RGB cv::Mat rgbFrame; @@ -50,7 +53,7 @@ image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame) } -void freeImageBuffer(image_buffer_t& imgBuffer) +void AiEngineInferenceOpi5::freeImageBuffer(image_buffer_t& imgBuffer) { if (imgBuffer.virt_addr) { delete[] imgBuffer.virt_addr; @@ -59,42 +62,25 @@ void freeImageBuffer(image_buffer_t& imgBuffer) } -cv::Mat resizeAndCenterImage(const cv::Mat& src, int targetWidth, int targetHeight) +cv::Mat AiEngineInferenceOpi5::resizeToHalfAndAssigntoTopLeft640x640(const cv::Mat& inputFrame) { - // Calculate the scaling factor and size - int originalWidth = src.cols; - int originalHeight = src.rows; - double aspectRatio = static_cast(originalWidth) / originalHeight; - int newWidth, newHeight; + // Resize input frame to half size + cv::Mat resizedFrame; + cv::resize(inputFrame, resizedFrame, cv::Size(), 0.5, 0.5); - if (originalWidth > originalHeight) { - newWidth = targetWidth; - newHeight = static_cast(targetWidth / aspectRatio); - } - else { - newHeight = targetHeight; - newWidth = static_cast(targetHeight * aspectRatio); - } + // Create a 640x640 frame to place the resized frame + cv::Mat outputFrame = cv::Mat::zeros(640, 640, inputFrame.type()); - // Resize the image - cv::Mat resizedImage; - cv::resize(src, resizedImage, cv::Size(newWidth, newHeight)); + // Copy the resized frame to the top-left corner of the output frame + cv::Rect roi(0, 0, resizedFrame.cols, resizedFrame.rows); + resizedFrame.copyTo(outputFrame(roi)); - // Create a black image of target size - cv::Mat outputImage = cv::Mat::zeros(targetHeight, targetWidth, src.type()); - - // Calculate position to center the image - int top = (targetHeight - newHeight) / 2; - int left = (targetWidth - newWidth) / 2; - - // Copy resized image into the center of the black image - resizedImage.copyTo(outputImage(cv::Rect(left, top, newWidth, newHeight))); - - return outputImage; + return outputFrame; } -void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) { +void AiEngineInferenceOpi5::drawObjects(cv::Mat& image, const object_detect_result_list& result_list) +{ for (int i = 0; i < result_list.count; i++) { const object_detect_result& result = result_list.results[i]; @@ -105,7 +91,6 @@ void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) { cv::rectangle(image, cv::Point(left, top), cv::Point(right, bottom), cv::Scalar(255, 0, 0), 2); - fprintf(stderr, "TUOMAS OMA. ID=%d String=%s\n", result.cls_id, coco_cls_to_name(result.cls_id)); // Text char c_text[256]; sprintf(c_text, "%s %.1f%%", coco_cls_to_name(result.cls_id), result.prop * 100); @@ -117,22 +102,32 @@ void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) { void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame) { - qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId(); + //qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId(); mActive = true; - cv::Mat scaledFrame = resizeAndCenterImage(frame, 640, 640); + cv::Mat scaledFrame = resizeToHalfAndAssigntoTopLeft640x640(frame); image_buffer_t imgBuffer = convertCV2FrameToImageBuffer(scaledFrame); object_detect_result_list od_results; int ret = inference_yolov8_model(&mRrknnAppCtx, &imgBuffer, &od_results); if (ret != 0) { qDebug() << "inference_yolov8_model() failure! ret: " << ret; + return; } + AiEngineInferenceResult result; for (int i = 0; i < od_results.count; i++) { object_detect_result *det_result = &(od_results.results[i]); - fprintf(stderr, "TUOMAS RKNN. ID=%d String=%s\n", det_result->cls_id, coco_cls_to_name(det_result->cls_id)); + + AiEngineObject object; + object.classId = det_result->cls_id; + object.propability = det_result->prop; + object.rectangle.top = det_result->box.top; + object.rectangle.left = det_result->box.left; + object.rectangle.bottom = det_result->box.bottom; + object.rectangle.right = det_result->box.right; + result.objects.append(object); } /* @@ -154,13 +149,6 @@ void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame) } */ - // Testing - - //write_image(filename.c_str(), &imgBuffer); - - drawObjects(scaledFrame, od_results); - freeImageBuffer(imgBuffer); - /* static int imageNum = 0; std::stringstream ss; @@ -170,9 +158,10 @@ void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame) cv::imwrite(filename, scaledFrame); */ - AiEngineInferenceResult result; + drawObjects(scaledFrame, od_results); + freeImageBuffer(imgBuffer); + result.frame = scaledFrame.clone(); - result.objects = od_results.count; emit resultsReady(result); mActive = false; diff --git a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h index 1dc4fbd..0804e9c 100644 --- a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h +++ b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h @@ -4,8 +4,6 @@ #include "aiengineinference.h" #include "yolov8.h" #include "image_utils.h" -#include "file_utils.h" -#include "image_drawing.h" class AiEngineInferenceOpi5 : public AiEngineInference { @@ -18,5 +16,10 @@ public slots: void performInferenceSlot(cv::Mat frame) override; private: + image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame); + void freeImageBuffer(image_buffer_t& imgBuffer); + cv::Mat resizeToHalfAndAssigntoTopLeft640x640(const cv::Mat& inputFrame); + void drawObjects(cv::Mat& image, const object_detect_result_list& result_list); + rknn_app_context_t mRrknnAppCtx; };