rewrite to zmq push and pull patterns.

file load works, suite can start up
This commit is contained in:
Alex Bezdieniezhnykh
2025-01-23 14:37:13 +02:00
parent ce25ef38b0
commit 82b3b526a7
20 changed files with 243 additions and 208 deletions
+22 -26
View File
@@ -1,11 +1,12 @@
import queue
import threading
from queue import Queue
cimport constants
import msgpack
from api_client cimport ApiClient
from annotation cimport Annotation
from inference import Inference
from processor_command cimport FileCommand, CommandType, ProcessorType
from remote_handlers cimport SocketHandler, RabbitHandler
from remote_command cimport RemoteCommand, CommandType
from remote_command_handler cimport RemoteCommandHandler
import argparse
cdef class ParsedArguments:
@@ -20,18 +21,15 @@ cdef class ParsedArguments:
cdef class CommandProcessor:
cdef ApiClient api_client
cdef SocketHandler socket_handler
cdef RabbitHandler rabbit_handler
cdef RemoteCommandHandler remote_handler
cdef object command_queue
cdef bint running
def __init__(self, args: ParsedArguments):
self.api_client = ApiClient(args.email, args.password, args.folder)
self.socket_handler = SocketHandler(self.on_message)
self.socket_handler.start()
self.rabbit_handler = RabbitHandler(self.api_client, self.on_message)
self.rabbit_handler.start()
self.command_queue = queue.Queue(maxsize=constants.QUEUE_MAXSIZE)
self.remote_handler = RemoteCommandHandler(self.on_command)
self.command_queue = Queue(maxsize=constants.QUEUE_MAXSIZE)
self.remote_handler.start()
self.running = True
def start(self):
@@ -44,25 +42,23 @@ cdef class CommandProcessor:
except Exception as e:
print(f"Error processing queue: {e}")
cdef on_message(self, FileCommand cmd):
cdef on_command(self, RemoteCommand command):
try:
if cmd.command_type == CommandType.INFERENCE:
self.command_queue.put(cmd)
elif cmd.command_type == CommandType.LOAD:
threading.Thread(target=self.process_load, args=[cmd], daemon=True).start()
if command.command_type == CommandType.INFERENCE:
self.command_queue.put(command)
elif command.command_type == CommandType.LOAD:
response = self.api_client.load_bytes(command.filename)
print(f'loaded file: {command.filename}, {len(response)} bytes')
self.remote_handler.send(response)
print(f'{len(response)} bytes was sent.')
except Exception as e:
print(f"Error handling client: {e}")
cdef on_annotations(self, cmd: FileCommand, annotations: [Annotation]):
handler = self.socket_handler if cmd.processor_type == ProcessorType.SOCKET else self.rabbit_handler
handler.send(annotations)
cdef process_load(self, FileCommand command):
response = self.api_client.load_bytes(command.filename)
handler = self.socket_handler if command.processor_type == ProcessorType.SOCKET else self.rabbit_handler
handler.send(response)
cdef on_annotations(self, RemoteCommand cmd, annotations: [Annotation]):
data = msgpack.packb(annotations)
self.remote_handler.send(data)
print(f'{len(data)} bytes was sent.')
def stop(self):
self.running = False