diff --git a/misc/rtsp_ai_player/aiengine.cpp b/misc/rtsp_ai_player/aiengine.cpp index 82a135c..9fa83f1 100644 --- a/misc/rtsp_ai_player/aiengine.cpp +++ b/misc/rtsp_ai_player/aiengine.cpp @@ -31,7 +31,7 @@ AiEngine::AiEngine(QString modelPath, QObject *parent) connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection); inferenceThread->start(); - mGimbalControl = new AiEngineGimbalControl(this); + mGimbalClient = new AiEngineGimbalClient(this); } @@ -50,11 +50,7 @@ void AiEngine::stop(void) void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult result) { //qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId(); - -#ifdef OPI5_BUILD - mGimbalControl->inferenceResultSlot(result); -#endif - + mGimbalClient->inferenceResultSlot(result); cv::imshow("Received Frame", result.frame); } diff --git a/misc/rtsp_ai_player/aiengine.h b/misc/rtsp_ai_player/aiengine.h index 8c9ddae..5362b5e 100644 --- a/misc/rtsp_ai_player/aiengine.h +++ b/misc/rtsp_ai_player/aiengine.h @@ -5,7 +5,7 @@ #include #include "aienginertsplistener.h" #include "aiengineinference.h" -#include "aienginegimbalcontrol.h" +#include "aienginegimbalclient.h" class AiEngine : public QObject { @@ -25,5 +25,5 @@ signals: private: AiEngineRtspListener *mRtspListener; AiEngineInference *mInference; - AiEngineGimbalControl *mGimbalControl; + AiEngineGimbalClient *mGimbalClient; }; diff --git a/misc/rtsp_ai_player/aienginedefinitions.h b/misc/rtsp_ai_player/aienginedefinitions.h new file mode 100644 index 0000000..2b70c05 --- /dev/null +++ b/misc/rtsp_ai_player/aienginedefinitions.h @@ -0,0 +1,50 @@ +#pragma once + +// Common geoposition +typedef struct { + float lat; + float lon; + float alt; +} AiEngineGeoPosition; + + +// AiEngineGimbalClient -> AiEngineGimbalServer is sent when location or position of the drone changes. +typedef struct { + AiEngineGeoPosition position; + float pitch; + float yaw; +} AiEngineDronePosition; + + +typedef struct { + int left; + int top; + int right; + int bottom; +} AiEngineRectangle; + + +// AiEngineGimbalClient -> AiEngineGimbalServer when AI finds the target. Index is a new variable to +// keep track of the targets. +typedef struct { + int index; + AiEngineRectangle rectangle; +} AiEngineCameraTarget; + + +// AiEngineGimbalServer -> AiEngineGimbalClient when AiEngineGimbalServer has zoomed to the target +// defined in AiEngineCameraTarget. Position variable contains calculated geoposition of the target. +typedef struct { + int targetIndex; + AiEngineGeoPosition position; +} AiEngineTargetPosition; + + +// AiEngineGimbalClient -> AiEngineGimbalServer when AI asks to change camera direction. For example +// nothing interesting was not found. AiEngineGimbalServer -> AiEngineGimbalClient every few second so +// that AI knows the position of the camera. +typedef struct { + float yaw; + float pitch; + float zoom; +} AiEngineCameraPosition; diff --git a/misc/rtsp_ai_player/aienginegimbalclient.cpp b/misc/rtsp_ai_player/aienginegimbalclient.cpp new file mode 100644 index 0000000..a536100 --- /dev/null +++ b/misc/rtsp_ai_player/aienginegimbalclient.cpp @@ -0,0 +1,113 @@ +#include +#include +#include +#include "aienginegimbalclient.h" +#include "aiengineinference.h" + +AiEngineGimbalClient::AiEngineGimbalClient(QObject *parent) + : QObject{parent} +{ + // Create server and run it in the new thread. + // Connect all signal and slots here. No need to do the same in AiEngineGimbalServer class. + + mGimbalServer = new AiEngineGimbalServer(this); + QThread *gimbalServerThread = new QThread(this); + mGimbalServer->moveToThread(gimbalServerThread); + + // Client -> Server communication. Only zoomToAiTarget() signal is emitted ATM. + connect(this, &AiEngineGimbalClient::setDronePosition, mGimbalServer, &AiEngineGimbalServer::dronePositionSlot, Qt::QueuedConnection); + connect(this, &AiEngineGimbalClient::zoomToAiTarget, mGimbalServer, &AiEngineGimbalServer::zoomToAiTargetSlot, Qt::QueuedConnection); + connect(this, &AiEngineGimbalClient::setCameraPosition, mGimbalServer, &AiEngineGimbalServer::cameraPositionSlot, Qt::QueuedConnection); + + // Server -> Client communication + connect(mGimbalServer, &AiEngineGimbalServer::aiTargetZoomed, this, &AiEngineGimbalClient::aiTargetZoomedSlot, Qt::QueuedConnection); + connect(mGimbalServer, &AiEngineGimbalServer::newCameraPosition, this, &AiEngineGimbalClient::cameraPositionSlot, Qt::QueuedConnection); + + gimbalServerThread->start(); +} + + +void AiEngineGimbalClient::aiTargetZoomedSlot(AiEngineTargetPosition targetPosition) +{ + qDebug() << "AiEngineGimbalClient::aiTargetZoomedSlot() Server zoomed to index:" + << targetPosition.targetIndex + << "Geopos:" + << targetPosition.position.lat + << targetPosition.position.lon + << targetPosition.position.alt; +} + + +void AiEngineGimbalClient::cameraPositionSlot(AiEngineCameraPosition cameraPosition) +{ + qDebug() << "AiEngineGimbalClient::cameraPositionSlot() Camera moved to:" + << cameraPosition.pitch + << cameraPosition.yaw + << cameraPosition.zoom; +} + + +AiEngineRectangle AiEngineGimbalClient::getGroupCoordinates(QVector &objects) +{ + AiEngineRectangle groupRectangle; + groupRectangle.top = 1000000; + groupRectangle.left = 1000000; + groupRectangle.bottom = 0; + groupRectangle.right = 0; + + for (int i = 0; i < objects.size(); i++) { + const AiEngineRectangle &objectRectangle = objects[i].rectangle; + + if (objectRectangle.top < groupRectangle.top) { + groupRectangle.top = objectRectangle.top; + } + if (objectRectangle.left < groupRectangle.left) { + groupRectangle.left = objectRectangle.left; + } + if (objectRectangle.bottom > groupRectangle.bottom) { + groupRectangle.bottom = objectRectangle.bottom; + } + if (objectRectangle.right > groupRectangle.right) { + groupRectangle.right = objectRectangle.right; + } + } + + return groupRectangle; +} + + +void AiEngineGimbalClient::inferenceResultSlot(AiEngineInferenceResult result) +{ + if (result.objects.size() == 0) { + return; + } + + // TODO!! Just increasing number for testing purposes ATM. + static int index = 0; + + // We got list of all recognized objects, but at least for now we will zoom to all objects at + // once and not for each invidually. Got minimal coordinates which contains the all objects. + AiEngineRectangle groupRect = getGroupCoordinates(result.objects); + + if (groupRect.top > 600 || groupRect.bottom > 600) { + qDebug() << "ERROR! inferenceResultSlot() groupRect.top > 600 || groupRect.bottom > 600"; + return; + } + + if (groupRect.left > 600 || groupRect.right > 600) { + qDebug() << "ERROR! inferenceResultSlot() groupRect.left > 600 || groupRect.right > 600"; + return; + } + + if ((groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)) { + qDebug() << "ERROR! (groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)"; + return; + } + + qDebug() << "inferenceResultSlot() Zooming to square top=" << groupRect.top << "x" << groupRect.left << "and bottom:" << groupRect.bottom << "x" << groupRect.right; + + AiEngineCameraTarget target; + target.rectangle = groupRect; + target.index = index++; + emit zoomToAiTarget(target); +} diff --git a/misc/rtsp_ai_player/aienginegimbalclient.h b/misc/rtsp_ai_player/aienginegimbalclient.h new file mode 100644 index 0000000..324cfc9 --- /dev/null +++ b/misc/rtsp_ai_player/aienginegimbalclient.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include "aiengineinference.h" +#include "aienginegimbalserver.h" +#include "aienginedefinitions.h" + +class AiEngineGimbalClient : public QObject +{ + Q_OBJECT +public: + explicit AiEngineGimbalClient(QObject *parent = nullptr); + +private slots: + void aiTargetZoomedSlot(AiEngineTargetPosition targetPosition); + void cameraPositionSlot(AiEngineCameraPosition cameraPosition); + +public slots: + void inferenceResultSlot(AiEngineInferenceResult results); + +signals: + void setDronePosition(AiEngineDronePosition); + void zoomToAiTarget(AiEngineCameraTarget); + void setCameraPosition(AiEngineCameraPosition); + +private: + AiEngineRectangle getGroupCoordinates(QVector &objects); + AiEngineGimbalServer *mGimbalServer; +}; diff --git a/misc/rtsp_ai_player/aienginegimbalcontrol.cpp b/misc/rtsp_ai_player/aienginegimbalcontrol.cpp deleted file mode 100644 index 6d4ab0c..0000000 --- a/misc/rtsp_ai_player/aienginegimbalcontrol.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include -#include "aienginegimbalcontrol.h" - -AiEngineGimbalControl::AiEngineGimbalControl(QObject *parent) - : QObject{parent} -{} - - -AiEngineRectangle AiEngineGimbalControl::getGroupCoordinates(QVector &objects) -{ - AiEngineRectangle groupRectangle; - groupRectangle.top = 1000000; - groupRectangle.left = 1000000; - groupRectangle.bottom = 0; - groupRectangle.right = 0; - - for (int i = 0; i < objects.size(); i++) { - const AiEngineRectangle &objectRectangle = objects[i].rectangle; - - if (objectRectangle.top < groupRectangle.top) { - groupRectangle.top = objectRectangle.top; - } - if (objectRectangle.left < groupRectangle.left) { - groupRectangle.left = objectRectangle.left; - } - if (objectRectangle.bottom > groupRectangle.bottom) { - groupRectangle.bottom = objectRectangle.bottom; - } - if (objectRectangle.right > groupRectangle.right) { - groupRectangle.right = objectRectangle.right; - } - } - - return groupRectangle; -} - - -void AiEngineGimbalControl::inferenceResultSlot(AiEngineInferenceResult result) -{ - if (result.objects.size() == 0) { - return; - } - - // We got list of all recognized objects, but at least for now we will zoom to all objects at - // once and not for each invidually. Got minimal coordinates which contains the all objects. - AiEngineRectangle groupRect = getGroupCoordinates(result.objects); - - // AI did inference with 640x360 resolution. Scale back to A8's 1280x720 resolution. - groupRect.top *= 2; - groupRect.left *= 2; - groupRect.bottom *= 2; - groupRect.right *= 2; - - if (groupRect.top > 720 || groupRect.bottom > 720) { - qDebug() << "ERROR! inferenceResultSlot() groupRect.top > 720 || groupRect.bottom > 720"; - return; - } - - if (groupRect.left > 1280 || groupRect.right > 1280) { - qDebug() << "ERROR! inferenceResultSlot() groupRect.left > 1280 || groupRect.right > 1280"; - return; - } - - if ((groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)) { - qDebug() << "ERROR! (groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)"; - return; - } - - qDebug() << "TUOMAS!! Zooming to square top=" << groupRect.top << "x" << groupRect.left << "and bottom:" << groupRect.bottom << "x" << groupRect.right; - - mRemoteControl.sendData(groupRect.top, groupRect.left, groupRect.bottom, groupRect.right); -} diff --git a/misc/rtsp_ai_player/aienginegimbalcontrol.h b/misc/rtsp_ai_player/aienginegimbalcontrol.h deleted file mode 100644 index 610af3f..0000000 --- a/misc/rtsp_ai_player/aienginegimbalcontrol.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include -#include -#include "aiengineinference.h" -#include "remoteControl.hpp" - -class AiEngineGimbalControl : public QObject -{ - Q_OBJECT -public: - explicit AiEngineGimbalControl(QObject *parent = nullptr); - -private: - AiEngineRectangle getGroupCoordinates(QVector &objects); - RemoteControl mRemoteControl; - -public slots: - void inferenceResultSlot(AiEngineInferenceResult results); -}; diff --git a/misc/rtsp_ai_player/aienginegimbalserver.cpp b/misc/rtsp_ai_player/aienginegimbalserver.cpp new file mode 100644 index 0000000..dcbfa31 --- /dev/null +++ b/misc/rtsp_ai_player/aienginegimbalserver.cpp @@ -0,0 +1,36 @@ +#include +#include "aienginegimbalserver.h" + +AiEngineGimbalServer::AiEngineGimbalServer(QObject *parent) + : QObject{parent} +{ + mSerialPort = new QSerialPort(this); + // TODO!! Setup and use serial port.... +} + + +// TODO!! Client doesn't really send any signal yet to this slot. +void AiEngineGimbalServer::dronePositionSlot(AiEngineDronePosition position) +{ + qDebug() << "AiEngineGimbalServer::dronePositionSlot() Server got new drone position:" + << position.position.lat + << position.position.lon + << position.position.alt; +} + + +// This is actually called from the client. +void AiEngineGimbalServer::zoomToAiTargetSlot(AiEngineCameraTarget target) +{ + qDebug() << "AiEngineGimbalServer::zoomToAiTargetSlot() Move camera to the new target:" + << "index:" << target.index + << "pos:" << target.rectangle.top << target.rectangle.left << target.rectangle.bottom << target.rectangle.right; + +} + + +// TODO!! Not sent from the client yet. +void AiEngineGimbalServer::cameraPositionSlot(AiEngineCameraPosition position) +{ + qDebug() << "SAiEngineGimbalServer::cameraPositionSlot() Move camera to:" << position.pitch << position.yaw << "zoom:" << position.zoom; +} diff --git a/misc/rtsp_ai_player/aienginegimbalserver.h b/misc/rtsp_ai_player/aienginegimbalserver.h new file mode 100644 index 0000000..d5b73c7 --- /dev/null +++ b/misc/rtsp_ai_player/aienginegimbalserver.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include "aienginedefinitions.h" + +class AiEngineGimbalServer : public QObject +{ + Q_OBJECT +public: + explicit AiEngineGimbalServer(QObject *parent = nullptr); + +public slots: + void dronePositionSlot(AiEngineDronePosition); + void zoomToAiTargetSlot(AiEngineCameraTarget); + void cameraPositionSlot(AiEngineCameraPosition); + +signals: + void aiTargetZoomed(AiEngineTargetPosition); + void newCameraPosition(AiEngineCameraPosition); + +private: + QSerialPort *mSerialPort; +}; diff --git a/misc/rtsp_ai_player/aiengineinference.h b/misc/rtsp_ai_player/aiengineinference.h index 8bf708d..9bfbf8f 100644 --- a/misc/rtsp_ai_player/aiengineinference.h +++ b/misc/rtsp_ai_player/aiengineinference.h @@ -5,21 +5,13 @@ #include #include #include +#include "aienginedefinitions.h" const int INFERENCE_SQUARE_WIDTH = 640; const int INFERENCE_SQUARE_HEIGHT = 640; -class AiEngineRectangle { -public: - int left; - int top; - int right; - int bottom; -}; - - class AiEngineObject { public: AiEngineRectangle rectangle; diff --git a/misc/rtsp_ai_player/rtsp_ai_player.pro b/misc/rtsp_ai_player/rtsp_ai_player.pro index c899769..ce2b477 100644 --- a/misc/rtsp_ai_player/rtsp_ai_player.pro +++ b/misc/rtsp_ai_player/rtsp_ai_player.pro @@ -1,4 +1,4 @@ -QT += core network +QT += core network serialport QT -= gui CONFIG += c++11 link_pkgconfig concurrent console PKGCONFIG += opencv4 @@ -6,7 +6,8 @@ MOC_DIR = moc OBJECTS_DIR = obj SOURCES = $$PWD/*.cpp $$PWD/../../misc/camera/a8_remote/remoteControl.cpp -HEADERS = $$PWD/*.h +HEADERS = $$PWD/*.h \ + aienginedefinitions.h INCLUDEPATH += $$PWD/../../misc/camera/a8_remote opi5 { diff --git a/misc/rtsp_ai_player/src-opencv-onnx/aiengineinferenceopencvonnx.cpp b/misc/rtsp_ai_player/src-opencv-onnx/aiengineinferenceopencvonnx.cpp index 9c88b3d..96a0291 100644 --- a/misc/rtsp_ai_player/src-opencv-onnx/aiengineinferenceopencvonnx.cpp +++ b/misc/rtsp_ai_player/src-opencv-onnx/aiengineinferenceopencvonnx.cpp @@ -16,7 +16,7 @@ AiEngineInferenceOpencvOnnx::AiEngineInferenceOpencvOnnx(QString modelPath, QObj void AiEngineInferenceOpencvOnnx::performInferenceSlot(cv::Mat frame) { try { - //qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId(); + qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId(); mActive = true;