diff --git a/tmp/opi_rtsp/aiengine.cpp b/tmp/opi_rtsp/aiengine.cpp index d71a898..a4263f6 100644 --- a/tmp/opi_rtsp/aiengine.cpp +++ b/tmp/opi_rtsp/aiengine.cpp @@ -2,8 +2,12 @@ #include #include "aiengine.h" #include "aiengineinference.h" -#include "src-onnx/aiengineinferenceonnx.h" +#ifdef OPI5_BUILD +#include "src-opi5/aiengineinferenceopi5.h" +#else +#include "src-onnx/aiengineinferenceonnx.h" +#endif AiEngine::AiEngine(QString modelPath, QObject *parent) : QObject{parent} @@ -11,8 +15,13 @@ AiEngine::AiEngine(QString modelPath, QObject *parent) mRtspListener = new AiEngineRtspListener(this); connect(mRtspListener, &AiEngineRtspListener::frameReceived, this, &AiEngine::frameReceivedSlot); +#ifdef OPI5_BUILD + mInference = new AiEngineInferenceOpi5(modelPath); +#else + mInference = new AiEngineInferenceOnnx(modelPath); +#endif + QThread *inferenceThread = new QThread(this); - mInference = new AiEngineInferenceOnnx(modelPath); mInference->moveToThread(inferenceThread); connect(mInference, &AiEngineInference::resultsReady, this, &AiEngine::inferenceResultsReceivedSlot, Qt::QueuedConnection); connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection); @@ -36,7 +45,12 @@ void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult results) { (void)results; qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId(); + cv::imshow("Received Frame", results.frame); + +//#ifndef OPI5_BUILD + //cv::imshow("Received Frame", results.frame); +//#endif } diff --git a/tmp/opi_rtsp/azaion_10_labels_list.txt b/tmp/opi_rtsp/azaion_10_labels_list.txt new file mode 100644 index 0000000..e7f4d2e --- /dev/null +++ b/tmp/opi_rtsp/azaion_10_labels_list.txt @@ -0,0 +1,10 @@ +Armoured_vehicle +Truck +Vehicle +Artillery +Shadow_artillery +Tranches +Military man +Tyre_tracks +Additional_protection_tank +Smoke diff --git a/tmp/opi_rtsp/config.h b/tmp/opi_rtsp/config.h index d6f91bf..435407e 100644 --- a/tmp/opi_rtsp/config.h +++ b/tmp/opi_rtsp/config.h @@ -2,4 +2,9 @@ #include +#ifdef OPI5_BUILD +//QString rtspVideoUrl = "rtsp://192.168.168.11:8554/live.stream"; +QString rtspVideoUrl = "rtsp://192.168.0.25:8554/main.264"; +#else QString rtspVideoUrl = "rtsp://localhost:8554/live.stream"; +#endif diff --git a/tmp/opi_rtsp/main.cpp b/tmp/opi_rtsp/main.cpp index 55c6f6c..0394725 100644 --- a/tmp/opi_rtsp/main.cpp +++ b/tmp/opi_rtsp/main.cpp @@ -20,8 +20,6 @@ int main(int argc, char *argv[]) return 1; } - cv::namedWindow("AiEngine", cv::WINDOW_AUTOSIZE); - AiEngine aiEngine(modelFile); aiEngine.start(); diff --git a/tmp/opi_rtsp/opi_rtsp.pro b/tmp/opi_rtsp/opi_rtsp.pro index 072d3fa..b2e283c 100644 --- a/tmp/opi_rtsp/opi_rtsp.pro +++ b/tmp/opi_rtsp/opi_rtsp.pro @@ -2,8 +2,10 @@ QT += core CONFIG += c++11 link_pkgconfig concurrent PKGCONFIG += opencv4 -SOURCES += *.cpp -HEADERS += *.h +SOURCES += $$PWD/*.cpp +HEADERS += $$PWD/*.h + +MOC_DIR = moc opi5 { message("OPI5 build") @@ -12,10 +14,9 @@ opi5 { QMAKE_CXXFLAGS += -DOPI5_BUILD LIBS += /usr/local/lib/librknnrt.so OBJECTS_DIR = objs-opi5 - SOURCES += src-opi5/*.c src-opi5/*.cc - HEADERS += src-opi5/*.h -} -else { + SOURCES += $$PWD/src-opi5/*.c $$PWD/src-opi5/*.cpp $$PWD/src-opi5/*.cc + HEADERS += $$PWD/src-opi5/*.h +} else { message("ONNX build") message("You must use YOLOv10 ONNX files") QMAKE_CXXFLAGS += -DONNX_BUILD @@ -23,6 +24,6 @@ else { LIBS += /opt/onnxruntime-linux-x64-1.18.0/lib/libonnxruntime.so.1.18.0 QMAKE_LFLAGS += -Wl,-rpath,/opt/onnxruntime-linux-x64-1.18.0/lib OBJECTS_DIR = objs-onnx - SOURCES += src-onnx/*.cpp - HEADERS += src-onnx/*.h + SOURCES += $$PWD/src-onnx/*.cpp + HEADERS += $$PWD/src-onnx/*.h } diff --git a/tmp/opi_rtsp/src-onnx/aiengineinferenceonnx.cpp b/tmp/opi_rtsp/src-onnx/aiengineinferenceonnx.cpp index 17c68a5..9242aa6 100644 --- a/tmp/opi_rtsp/src-onnx/aiengineinferenceonnx.cpp +++ b/tmp/opi_rtsp/src-onnx/aiengineinferenceonnx.cpp @@ -1,7 +1,5 @@ -#include #include -#include -#include +#include #include "aiengineinferenceonnx.h" AiEngineInferenceOnnx::AiEngineInferenceOnnx(QString modelPath, QObject *parent) diff --git a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp new file mode 100644 index 0000000..0f6124d --- /dev/null +++ b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include +#include "aiengineinferenceopi5.h" + +AiEngineInferenceOpi5::AiEngineInferenceOpi5(QString modelPath, QObject *parent) + : AiEngineInference{modelPath, parent} +{ + qDebug() << "AiEngineInferenceOpi5() test mModelPath=" << mModelPath; + + memset(&mRrknnAppCtx, 0, sizeof(rknn_app_context_t)); + init_post_process(); + + int ret = init_yolov8_model(modelPath.toLocal8Bit(), &mRrknnAppCtx); + if (ret != 0) { + qDebug() << "init_yolov8_model() failure! ret: " << ret << "modelPath = " << modelPath; + return; + } +} + + +AiEngineInferenceOpi5::~AiEngineInferenceOpi5() +{ + deinit_post_process(); + release_yolov8_model(&mRrknnAppCtx); +} + + +image_buffer_t convertCV2FrameToImageBuffer(const cv::Mat& bgrFrame) +{ + // Convert BGR to RGB + cv::Mat rgbFrame; + cv::cvtColor(bgrFrame, rgbFrame, cv::COLOR_BGR2RGB); + + image_buffer_t imgBuffer; + memset(&imgBuffer, 0, sizeof(image_buffer_t)); + imgBuffer.width = rgbFrame.cols; + imgBuffer.height = rgbFrame.rows; + imgBuffer.width_stride = rgbFrame.step; + imgBuffer.height_stride = rgbFrame.rows; + imgBuffer.format = IMAGE_FORMAT_RGB888; + imgBuffer.size = rgbFrame.total() * rgbFrame.elemSize(); + + imgBuffer.virt_addr = new unsigned char[imgBuffer.size]; + std::memcpy(imgBuffer.virt_addr, rgbFrame.data, imgBuffer.size); + + return imgBuffer; +} + + +void freeImageBuffer(image_buffer_t& imgBuffer) +{ + if (imgBuffer.virt_addr) { + delete[] imgBuffer.virt_addr; + imgBuffer.virt_addr = nullptr; + } +} + + +cv::Mat resizeAndCenterImage(const cv::Mat& src, int targetWidth, int targetHeight) +{ + // Calculate the scaling factor and size + int originalWidth = src.cols; + int originalHeight = src.rows; + double aspectRatio = static_cast(originalWidth) / originalHeight; + int newWidth, newHeight; + + if (originalWidth > originalHeight) { + newWidth = targetWidth; + newHeight = static_cast(targetWidth / aspectRatio); + } + else { + newHeight = targetHeight; + newWidth = static_cast(targetHeight * aspectRatio); + } + + // Resize the image + cv::Mat resizedImage; + cv::resize(src, resizedImage, cv::Size(newWidth, newHeight)); + + // Create a black image of target size + cv::Mat outputImage = cv::Mat::zeros(targetHeight, targetWidth, src.type()); + + // Calculate position to center the image + int top = (targetHeight - newHeight) / 2; + int left = (targetWidth - newWidth) / 2; + + // Copy resized image into the center of the black image + resizedImage.copyTo(outputImage(cv::Rect(left, top, newWidth, newHeight))); + + return outputImage; +} + + +void drawObjects(cv::Mat& image, const object_detect_result_list& result_list) { + for (int i = 0; i < result_list.count; i++) { + const object_detect_result& result = result_list.results[i]; + + int left = result.box.left; + int top = result.box.top; + int right = result.box.right; + int bottom = result.box.bottom; + + cv::rectangle(image, cv::Point(left, top), cv::Point(right, bottom), cv::Scalar(255, 0, 0), 2); + + fprintf(stderr, "TUOMAS OMA. ID=%d String=%s\n", result.cls_id, coco_cls_to_name(result.cls_id)); + // Text + char c_text[256]; + sprintf(c_text, "%s %.1f%%", coco_cls_to_name(result.cls_id), result.prop * 100); + cv::Point textOrg(left, top - 5); + cv::putText(image, std::string(c_text), textOrg, cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 255), 1); + } +} + + +void AiEngineInferenceOpi5::performInferenceSlot(cv::Mat frame) +{ + qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId(); + + mActive = true; + + cv::Mat scaledFrame = resizeAndCenterImage(frame, 640, 640); + image_buffer_t imgBuffer = convertCV2FrameToImageBuffer(scaledFrame); + + object_detect_result_list od_results; + int ret = inference_yolov8_model(&mRrknnAppCtx, &imgBuffer, &od_results); + if (ret != 0) { + qDebug() << "inference_yolov8_model() failure! ret: " << ret; + } + + for (int i = 0; i < od_results.count; i++) { + object_detect_result *det_result = &(od_results.results[i]); + fprintf(stderr, "TUOMAS RKNN. ID=%d String=%s\n", det_result->cls_id, coco_cls_to_name(det_result->cls_id)); + } + + /* + char text[256]; + for (int i = 0; i < od_results.count; i++) { + object_detect_result *det_result = &(od_results.results[i]); + printf("%s @ (%d %d %d %d) %.3f\n", coco_cls_to_name(det_result->cls_id), + det_result->box.left, det_result->box.top, + det_result->box.right, det_result->box.bottom, + det_result->prop); + int x1 = det_result->box.left; + int y1 = det_result->box.top; + int x2 = det_result->box.right; + int y2 = det_result->box.bottom; + + draw_rectangle(&imgBuffer, x1, y1, x2 - x1, y2 - y1, COLOR_BLUE, 3); + sprintf(text, "%s %.1f%%", coco_cls_to_name(det_result->cls_id), det_result->prop * 100); + draw_text(&imgBuffer, text, x1, y1 - 20, COLOR_RED, 10); + } + */ + + // Testing + + //write_image(filename.c_str(), &imgBuffer); + + drawObjects(scaledFrame, od_results); + freeImageBuffer(imgBuffer); + + /* + static int imageNum = 0; + std::stringstream ss; + ss << std::setw(4) << std::setfill('0') << imageNum++; + std::string formatted_number = ss.str(); + std::string filename = "/tmp/out-" + formatted_number + ".png"; + cv::imwrite(filename, scaledFrame); + */ + + AiEngineInferenceResult result; + result.frame = scaledFrame.clone(); + result.objects = od_results.count; + emit resultsReady(result); + + mActive = false; +} diff --git a/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h new file mode 100644 index 0000000..1dc4fbd --- /dev/null +++ b/tmp/opi_rtsp/src-opi5/aiengineinferenceopi5.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include "aiengineinference.h" +#include "yolov8.h" +#include "image_utils.h" +#include "file_utils.h" +#include "image_drawing.h" + +class AiEngineInferenceOpi5 : public AiEngineInference +{ + Q_OBJECT +public: + explicit AiEngineInferenceOpi5(QString modelPath, QObject *parent = nullptr); + ~AiEngineInferenceOpi5(); + +public slots: + void performInferenceSlot(cv::Mat frame) override; + +private: + rknn_app_context_t mRrknnAppCtx; +}; diff --git a/tmp/opi_rtsp/src-opi5/postprocess.cc b/tmp/opi_rtsp/src-opi5/postprocess.cc index 729b398..a64d70f 100644 --- a/tmp/opi_rtsp/src-opi5/postprocess.cc +++ b/tmp/opi_rtsp/src-opi5/postprocess.cc @@ -23,7 +23,13 @@ #include #include -#define LABEL_NALE_TXT_PATH "./model/coco_80_labels_list.txt" + +#ifdef OPI5_BUILD +#define LABEL_NALE_TXT_PATH "azaion_10_labels_list.txt" +#else +#define LABEL_NALE_TXT_PATH "coco_80_labels_list.txt" +#endif + static char *labels[OBJ_CLASS_NUM];