Initial version of combining rtsp_ai_player and camera modules.

Functionality has been written to rtsp_ai_player.

TODO!!
- move functionality of camera module misc/rtsp_ai_player/aienginegimbalserver.cpp
- implement all signals in AiEngineGimbalClient
- get drone position from autopilot and send it to AiEngineGimbalServer

misc/rtsp_ai_player/aiengine.cpp

@@ -31,7 +31,7 @@ AiEngine::AiEngine(QString modelPath, QObject *parent)
     connect(this, &AiEngine::inferenceFrame, mInference, &AiEngineInference::performInferenceSlot, Qt::QueuedConnection);
     inferenceThread->start();
 
-    mGimbalControl =  new AiEngineGimbalControl(this);
+    mGimbalClient = new AiEngineGimbalClient(this);
 }
 
 
@@ -50,11 +50,7 @@ void AiEngine::stop(void)
 void AiEngine::inferenceResultsReceivedSlot(AiEngineInferenceResult result)
 {
     //qDebug() << "AiEngine got inference results in thread: " << QThread::currentThreadId();
-
+    mGimbalClient->inferenceResultSlot(result);
-#ifdef OPI5_BUILD
-    mGimbalControl->inferenceResultSlot(result);
-#endif
-
     cv::imshow("Received Frame", result.frame);
 }
 

misc/rtsp_ai_player/aiengine.h

@@ -5,7 +5,7 @@
 #include <opencv2/videoio.hpp>
 #include "aienginertsplistener.h"
 #include "aiengineinference.h"
-#include "aienginegimbalcontrol.h"
+#include "aienginegimbalclient.h"
 
 class AiEngine : public QObject
 {
@@ -25,5 +25,5 @@ signals:
 private:
     AiEngineRtspListener *mRtspListener;
     AiEngineInference *mInference;
-    AiEngineGimbalControl *mGimbalControl;
+    AiEngineGimbalClient *mGimbalClient;
 };

misc/rtsp_ai_player/aienginedefinitions.h

@@ -0,0 +1,50 @@
+#pragma once
+
+// Common geoposition
+typedef struct {
+    float lat;
+    float lon;
+    float alt;
+} AiEngineGeoPosition;
+
+
+// AiEngineGimbalClient -> AiEngineGimbalServer is sent when location or position of the drone changes.
+typedef struct {
+    AiEngineGeoPosition position;
+    float pitch;
+    float yaw;
+} AiEngineDronePosition;
+
+
+typedef struct {
+    int left;
+    int top;
+    int right;
+    int bottom;
+} AiEngineRectangle;
+
+
+// AiEngineGimbalClient -> AiEngineGimbalServer when AI finds the target. Index is a new variable to
+// keep track of the targets.
+typedef struct {
+    int index;
+    AiEngineRectangle rectangle;
+} AiEngineCameraTarget;
+
+
+// AiEngineGimbalServer -> AiEngineGimbalClient when AiEngineGimbalServer has zoomed to the target
+// defined in AiEngineCameraTarget. Position variable contains calculated geoposition of the target.
+typedef struct {
+    int targetIndex;
+    AiEngineGeoPosition position;
+} AiEngineTargetPosition;
+
+
+// AiEngineGimbalClient -> AiEngineGimbalServer when AI asks to change camera direction. For example
+// nothing interesting was not found. AiEngineGimbalServer -> AiEngineGimbalClient every few second so
+// that AI knows the position of the camera.
+typedef struct {
+    float yaw;
+    float pitch;
+    float zoom;
+} AiEngineCameraPosition;

misc/rtsp_ai_player/aienginegimbalclient.cpp

@@ -0,0 +1,113 @@
+#include <QDebug>
+#include <QThread>
+#include <QVector>
+#include "aienginegimbalclient.h"
+#include "aiengineinference.h"
+
+AiEngineGimbalClient::AiEngineGimbalClient(QObject *parent)
+    : QObject{parent}
+{
+    // Create server and run it in the new thread.
+    // Connect all signal and slots here. No need to do the same in AiEngineGimbalServer class.
+
+    mGimbalServer = new AiEngineGimbalServer(this);
+    QThread *gimbalServerThread = new QThread(this);
+    mGimbalServer->moveToThread(gimbalServerThread);
+
+    // Client -> Server communication. Only zoomToAiTarget() signal is emitted ATM.
+    connect(this, &AiEngineGimbalClient::setDronePosition,  mGimbalServer, &AiEngineGimbalServer::dronePositionSlot,  Qt::QueuedConnection);
+    connect(this, &AiEngineGimbalClient::zoomToAiTarget,    mGimbalServer, &AiEngineGimbalServer::zoomToAiTargetSlot, Qt::QueuedConnection);
+    connect(this, &AiEngineGimbalClient::setCameraPosition, mGimbalServer, &AiEngineGimbalServer::cameraPositionSlot, Qt::QueuedConnection);
+
+    // Server -> Client communication
+    connect(mGimbalServer, &AiEngineGimbalServer::aiTargetZoomed,    this, &AiEngineGimbalClient::aiTargetZoomedSlot, Qt::QueuedConnection);
+    connect(mGimbalServer, &AiEngineGimbalServer::newCameraPosition, this, &AiEngineGimbalClient::cameraPositionSlot, Qt::QueuedConnection);
+
+    gimbalServerThread->start();
+}
+
+
+void AiEngineGimbalClient::aiTargetZoomedSlot(AiEngineTargetPosition targetPosition)
+{
+    qDebug() << "AiEngineGimbalClient::aiTargetZoomedSlot() Server zoomed to index:"
+             << targetPosition.targetIndex
+             <<  "Geopos:"
+             << targetPosition.position.lat
+             << targetPosition.position.lon
+             << targetPosition.position.alt;
+}
+
+
+void AiEngineGimbalClient::cameraPositionSlot(AiEngineCameraPosition cameraPosition)
+{
+    qDebug() << "AiEngineGimbalClient::cameraPositionSlot() Camera moved to:"
+             << cameraPosition.pitch
+             << cameraPosition.yaw
+             << cameraPosition.zoom;
+}
+
+
+AiEngineRectangle AiEngineGimbalClient::getGroupCoordinates(QVector<AiEngineObject> &objects)
+{
+    AiEngineRectangle groupRectangle;
+    groupRectangle.top    = 1000000;
+    groupRectangle.left   = 1000000;
+    groupRectangle.bottom = 0;
+    groupRectangle.right  = 0;
+
+    for (int i = 0; i < objects.size(); i++) {
+        const AiEngineRectangle &objectRectangle = objects[i].rectangle;
+
+        if (objectRectangle.top < groupRectangle.top) {
+            groupRectangle.top = objectRectangle.top;
+        }
+        if (objectRectangle.left < groupRectangle.left) {
+            groupRectangle.left = objectRectangle.left;
+        }
+        if (objectRectangle.bottom > groupRectangle.bottom) {
+            groupRectangle.bottom = objectRectangle.bottom;
+        }
+        if (objectRectangle.right > groupRectangle.right) {
+            groupRectangle.right = objectRectangle.right;
+        }
+    }
+
+    return groupRectangle;
+}
+
+
+void AiEngineGimbalClient::inferenceResultSlot(AiEngineInferenceResult result)
+{
+    if (result.objects.size() == 0) {
+        return;
+    }
+
+    // TODO!! Just increasing number for testing purposes ATM.
+    static int index = 0;
+
+    // We got list of all recognized objects, but at least for now we will zoom to all objects at
+    // once and not for each invidually. Got minimal coordinates which contains the all objects.
+    AiEngineRectangle groupRect = getGroupCoordinates(result.objects);
+
+    if (groupRect.top > 600 || groupRect.bottom > 600) {
+        qDebug() << "ERROR! inferenceResultSlot() groupRect.top > 600 || groupRect.bottom > 600";
+        return;
+    }
+
+    if (groupRect.left > 600 || groupRect.right > 600) {
+        qDebug() << "ERROR! inferenceResultSlot() groupRect.left > 600 || groupRect.right > 600";
+        return;
+    }
+
+    if ((groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)) {
+        qDebug() << "ERROR! (groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)";
+        return;
+    }
+
+    qDebug() << "inferenceResultSlot() Zooming to square top=" << groupRect.top << "x" << groupRect.left << "and bottom:" << groupRect.bottom << "x" << groupRect.right;
+
+    AiEngineCameraTarget target;
+    target.rectangle = groupRect;
+    target.index     = index++;
+    emit zoomToAiTarget(target);
+}

misc/rtsp_ai_player/aienginegimbalclient.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <QObject>
+#include <QElapsedTimer>
+#include "aiengineinference.h"
+#include "aienginegimbalserver.h"
+#include "aienginedefinitions.h"
+
+class AiEngineGimbalClient : public QObject
+{
+    Q_OBJECT
+public:
+    explicit AiEngineGimbalClient(QObject *parent = nullptr);
+
+private slots:
+    void aiTargetZoomedSlot(AiEngineTargetPosition targetPosition);
+    void cameraPositionSlot(AiEngineCameraPosition cameraPosition);
+
+public slots:
+    void inferenceResultSlot(AiEngineInferenceResult results);
+
+signals:
+    void setDronePosition(AiEngineDronePosition);
+    void zoomToAiTarget(AiEngineCameraTarget);
+    void setCameraPosition(AiEngineCameraPosition);
+
+private:
+    AiEngineRectangle getGroupCoordinates(QVector<AiEngineObject> &objects);
+    AiEngineGimbalServer *mGimbalServer;
+};

misc/rtsp_ai_player/aienginegimbalcontrol.cpp

@@ -1,73 +0,0 @@
-#include <QDebug>
-#include <QVector>
-#include "aienginegimbalcontrol.h"
-
-AiEngineGimbalControl::AiEngineGimbalControl(QObject *parent)
-    : QObject{parent}
-{}
-
-
-AiEngineRectangle AiEngineGimbalControl::getGroupCoordinates(QVector<AiEngineObject> &objects)
-{
-    AiEngineRectangle groupRectangle;
-    groupRectangle.top    = 1000000;
-    groupRectangle.left   = 1000000;
-    groupRectangle.bottom = 0;
-    groupRectangle.right  = 0;
-
-    for (int i = 0; i < objects.size(); i++) {
-        const AiEngineRectangle &objectRectangle = objects[i].rectangle;
-
-        if (objectRectangle.top < groupRectangle.top) {
-            groupRectangle.top = objectRectangle.top;
-        }
-        if (objectRectangle.left < groupRectangle.left) {
-            groupRectangle.left = objectRectangle.left;
-        }
-        if (objectRectangle.bottom > groupRectangle.bottom) {
-            groupRectangle.bottom = objectRectangle.bottom;
-        }
-        if (objectRectangle.right > groupRectangle.right) {
-            groupRectangle.right = objectRectangle.right;
-        }
-    }
-
-    return groupRectangle;
-}
-
-
-void AiEngineGimbalControl::inferenceResultSlot(AiEngineInferenceResult result)
-{
-    if (result.objects.size() == 0) {
-        return;
-    }
-
-    // We got list of all recognized objects, but at least for now we will zoom to all objects at
-    // once and not for each invidually. Got minimal coordinates which contains the all objects.
-    AiEngineRectangle groupRect = getGroupCoordinates(result.objects);
-
-    // AI did inference with 640x360 resolution. Scale back to A8's 1280x720 resolution.
-    groupRect.top    *= 2;
-    groupRect.left   *= 2;
-    groupRect.bottom *= 2;
-    groupRect.right  *= 2;
-
-    if (groupRect.top > 720 || groupRect.bottom > 720) {
-        qDebug() << "ERROR! inferenceResultSlot() groupRect.top > 720 || groupRect.bottom > 720";
-        return;
-    }
-
-    if (groupRect.left > 1280 || groupRect.right > 1280) {
-        qDebug() << "ERROR! inferenceResultSlot() groupRect.left > 1280 || groupRect.right > 1280";
-        return;
-    }
-
-    if ((groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)) {
-        qDebug() << "ERROR! (groupRect.bottom <= groupRect.top) || (groupRect.right <= groupRect.left)";
-        return;
-    }
-
-    qDebug() << "TUOMAS!! Zooming to square top=" << groupRect.top << "x" << groupRect.left << "and bottom:" << groupRect.bottom << "x" << groupRect.right;
-
-    mRemoteControl.sendData(groupRect.top, groupRect.left, groupRect.bottom, groupRect.right);
-}

misc/rtsp_ai_player/aienginegimbalcontrol.h

@@ -1,20 +0,0 @@
-#pragma once
-
-#include <QObject>
-#include <QElapsedTimer>
-#include "aiengineinference.h"
-#include "remoteControl.hpp"
-
-class AiEngineGimbalControl : public QObject
-{
-    Q_OBJECT
-public:
-    explicit AiEngineGimbalControl(QObject *parent = nullptr);
-
-private:
-    AiEngineRectangle getGroupCoordinates(QVector<AiEngineObject> &objects);
-    RemoteControl mRemoteControl;
-
-public slots:
-    void inferenceResultSlot(AiEngineInferenceResult results);
-};

misc/rtsp_ai_player/aienginegimbalserver.cpp

@@ -0,0 +1,36 @@
+#include <QDebug>
+#include "aienginegimbalserver.h"
+
+AiEngineGimbalServer::AiEngineGimbalServer(QObject *parent)
+    : QObject{parent}
+{
+    mSerialPort = new QSerialPort(this);
+    // TODO!! Setup and use serial port....
+}
+
+
+// TODO!! Client doesn't really send any signal yet to this slot.
+void AiEngineGimbalServer::dronePositionSlot(AiEngineDronePosition position)
+{
+    qDebug() << "AiEngineGimbalServer::dronePositionSlot() Server got new drone position:"
+             << position.position.lat
+             << position.position.lon
+             << position.position.alt;
+}
+
+
+// This is actually called from the client.
+void AiEngineGimbalServer::zoomToAiTargetSlot(AiEngineCameraTarget target)
+{
+    qDebug() << "AiEngineGimbalServer::zoomToAiTargetSlot() Move camera to the new target:"
+             << "index:" << target.index
+             << "pos:" << target.rectangle.top << target.rectangle.left << target.rectangle.bottom << target.rectangle.right;
+
+}
+
+
+// TODO!! Not sent from the client yet.
+void AiEngineGimbalServer::cameraPositionSlot(AiEngineCameraPosition position)
+{
+    qDebug() << "SAiEngineGimbalServer::cameraPositionSlot() Move camera to:" << position.pitch <<  position.yaw << "zoom:" << position.zoom;
+}

misc/rtsp_ai_player/aienginegimbalserver.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <QObject>
+#include <QSerialPort>
+#include "aienginedefinitions.h"
+
+class AiEngineGimbalServer : public QObject
+{
+    Q_OBJECT
+public:
+    explicit AiEngineGimbalServer(QObject *parent = nullptr);
+
+public slots:
+    void dronePositionSlot(AiEngineDronePosition);
+    void zoomToAiTargetSlot(AiEngineCameraTarget);
+    void cameraPositionSlot(AiEngineCameraPosition);
+
+signals:
+    void aiTargetZoomed(AiEngineTargetPosition);
+    void newCameraPosition(AiEngineCameraPosition);
+
+private:
+    QSerialPort *mSerialPort;
+};

misc/rtsp_ai_player/aiengineinference.h

@@ -5,21 +5,13 @@
 #include <QVector>
 #include <opencv2/core.hpp>
 #include <opencv2/imgproc.hpp>
+#include "aienginedefinitions.h"
 
 
 const int INFERENCE_SQUARE_WIDTH  = 640;
 const int INFERENCE_SQUARE_HEIGHT = 640;
 
 
-class AiEngineRectangle {
-public:
-    int left;
-    int top;
-    int right;
-    int bottom;
-};
-
-
 class AiEngineObject {
 public:
     AiEngineRectangle rectangle;

misc/rtsp_ai_player/rtsp_ai_player.pro

@@ -1,4 +1,4 @@
-QT += core network
+QT += core network serialport
 QT -= gui
 CONFIG += c++11 link_pkgconfig concurrent console
 PKGCONFIG += opencv4
@@ -6,7 +6,8 @@ MOC_DIR = moc
 OBJECTS_DIR = obj
 
 SOURCES = $$PWD/*.cpp $$PWD/../../misc/camera/a8_remote/remoteControl.cpp
-HEADERS = $$PWD/*.h
+HEADERS = $$PWD/*.h \
+    aienginedefinitions.h
 INCLUDEPATH += $$PWD/../../misc/camera/a8_remote
 
 opi5 {

misc/rtsp_ai_player/src-opencv-onnx/aiengineinferenceopencvonnx.cpp

@@ -16,7 +16,7 @@ AiEngineInferenceOpencvOnnx::AiEngineInferenceOpencvOnnx(QString modelPath, QObj
 void AiEngineInferenceOpencvOnnx::performInferenceSlot(cv::Mat frame)
 {
     try {
-        //qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId();
+        qDebug() << "performInferenceSlot() in thread: " << QThread::currentThreadId();
 
         mActive = true;