Refactor configuration and update test structure for improved clarity

- Updated `.gitignore` to remove committed test fixture data exclusions.
- Increased batch size in `config.test.yaml` from 4 to 128 for training.
- Simplified directory structure in `config.yaml` by removing unnecessary data paths.
- Adjusted paths in `augmentation.py`, `dataset-visualiser.py`, and `exports.py` to align with the new configuration structure.
- Enhanced `annotation_queue_handler.py` to utilize the updated configuration for directory management.
- Added CSV logging of test results in `conftest.py` for better test reporting.

These changes streamline the configuration management and enhance the testing framework, ensuring better organization and clarity in the project.

src/annotation-queue/annotation_queue_handler.py

@@ -40,36 +40,26 @@ class AnnotationQueueHandler:
 
     class AnnotationName:
         def __init__(self, h, name):
-            self.img_data = path.join(h.data_dir, h.images_dir, f"{name}{h.JPG_EXT}")
-            self.lbl_data = path.join(h.data_dir, h.labels_dir, f"{name}{h.TXT_EXT}")
-            self.img_seed = path.join(h.data_seed_dir, h.images_dir, f"{name}{h.JPG_EXT}")
-            self.lbl_seed = path.join(h.data_seed_dir, h.labels_dir, f"{name}{h.TXT_EXT}")
+            self.img_data = path.join(h.cfg.images_dir, f"{name}{h.JPG_EXT}")
+            self.lbl_data = path.join(h.cfg.labels_dir, f"{name}{h.TXT_EXT}")
+            self.img_seed = path.join(h.cfg.seed_images_dir, f"{name}{h.JPG_EXT}")
+            self.lbl_seed = path.join(h.cfg.seed_labels_dir, f"{name}{h.TXT_EXT}")
 
     def __init__(self):
-        cfg = constants.config
-        self.data_dir = path.join(cfg.dirs.root, cfg.dirs.data)
-        self.data_seed_dir = path.join(cfg.dirs.root, cfg.dirs.data_seed)
-        self.images_dir = cfg.dirs.images
-        self.labels_dir = cfg.dirs.labels
+        self.cfg = constants.config
 
-        self.del_img_dir = path.join(cfg.dirs.root, cfg.dirs.data_deleted, self.images_dir)
-        self.del_lbl_dir = path.join(cfg.dirs.root, cfg.dirs.data_deleted, self.labels_dir)
-
-        makedirs(path.join(self.data_dir, self.images_dir), exist_ok=True)
-        makedirs(path.join(self.data_dir, self.labels_dir), exist_ok=True)
-        makedirs(path.join(self.data_seed_dir, self.images_dir), exist_ok=True)
-        makedirs(path.join(self.data_seed_dir, self.labels_dir), exist_ok=True)
-
-        makedirs(self.del_img_dir, exist_ok=True)
-        makedirs(self.del_lbl_dir, exist_ok=True)
+        for d in (self.cfg.images_dir, self.cfg.labels_dir,
+                  self.cfg.seed_images_dir, self.cfg.seed_labels_dir,
+                  self.cfg.del_images_dir, self.cfg.del_labels_dir):
+            makedirs(d, exist_ok=True)
 
         self.consumer = Consumer(
-            host=cfg.queue.host,
-            port=cfg.queue.port,
-            username=cfg.queue.consumer_user,
-            password=cfg.queue.consumer_pw
+            host=self.cfg.queue.host,
+            port=self.cfg.queue.port,
+            username=self.cfg.queue.consumer_user,
+            password=self.cfg.queue.consumer_pw
         )
-        self.queue_name = cfg.queue.name
+        self.queue_name = self.cfg.queue.name
 
         try:
             with open(self.OFFSET_FILE, 'r') as f:
@@ -154,14 +144,14 @@ class AnnotationQueueHandler:
         for name in msg.annotation_names:
             a = self.AnnotationName(self, name)
             if path.exists(a.img_data):
-                shutil.move(a.img_data, self.del_img_dir)
+                shutil.move(a.img_data, self.cfg.del_images_dir)
             if path.exists(a.img_seed):
-                shutil.move(a.img_seed, self.del_img_dir)
+                shutil.move(a.img_seed, self.cfg.del_images_dir)
 
             if path.exists(a.lbl_data):
-                shutil.move(a.lbl_data, self.del_lbl_dir)
+                shutil.move(a.lbl_data, self.cfg.del_labels_dir)
             if path.exists(a.lbl_seed):
-                shutil.move(a.lbl_seed, self.del_lbl_dir)
+                shutil.move(a.lbl_seed, self.cfg.del_labels_dir)
 
 
 if __name__ == '__main__':

src/augmentation.py

@@ -95,8 +95,8 @@ class Augmentator:
 
     def augment_annotation(self, image_file):
         try:
-            image_path = os.path.join(constants.config.data_images_dir, image_file.name)
-            labels_path = os.path.join(constants.config.data_labels_dir, f'{Path(str(image_path)).stem}.txt')
+            image_path = os.path.join(constants.config.images_dir, image_file.name)
+            labels_path = os.path.join(constants.config.labels_dir, f'{Path(str(image_path)).stem}.txt')
             image = cv2.imdecode(np.fromfile(image_path, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
 
             img_ann = ImageLabel(
@@ -134,7 +134,7 @@ class Augmentator:
 
         processed_images = set(f.name for f in os.scandir(constants.config.processed_images_dir))
         images = []
-        with os.scandir(constants.config.data_images_dir) as imd:
+        with os.scandir(constants.config.images_dir) as imd:
             for image_file in imd:
                 if image_file.is_file() and image_file.name not in processed_images:
                     images.append(image_file)

src/constants.py

@@ -24,10 +24,7 @@ class DirsConfig(BaseModel):
     root: str = '/azaion'
     data: str = 'data'
     data_seed: str = 'data-seed'
-    data_processed: str = 'data-processed'
     data_deleted: str = 'data_deleted'
-    images: str = 'images'
-    labels: str = 'labels'
 
 
 class TrainingConfig(BaseModel):
@@ -60,24 +57,36 @@ class Config(BaseModel):
         return path.join(self.dirs.root, self.dirs.data)
 
     @property
-    def data_images_dir(self) -> str:
-        return path.join(self.data_dir, self.dirs.images)
+    def images_dir(self) -> str:
+        return path.join(self.data_dir, IMAGES_DIR)
 
     @property
-    def data_labels_dir(self) -> str:
-        return path.join(self.data_dir, self.dirs.labels)
+    def labels_dir(self) -> str:
+        return path.join(self.data_dir, LABELS_DIR)
 
     @property
-    def processed_dir(self) -> str:
-        return path.join(self.dirs.root, self.dirs.data_processed)
+    def seed_dir(self) -> str:
+        return path.join(self.dirs.root, self.dirs.data_seed)
 
     @property
-    def processed_images_dir(self) -> str:
-        return path.join(self.processed_dir, self.dirs.images)
+    def seed_images_dir(self) -> str:
+        return path.join(self.seed_dir, IMAGES_DIR)
 
     @property
-    def processed_labels_dir(self) -> str:
-        return path.join(self.processed_dir, self.dirs.labels)
+    def seed_labels_dir(self) -> str:
+        return path.join(self.seed_dir, LABELS_DIR)
+
+    @property
+    def del_dir(self) -> str:
+        return path.join(self.dirs.root, self.dirs.data_deleted)
+
+    @property
+    def del_images_dir(self) -> str:
+        return path.join(self.del_dir, IMAGES_DIR)
+
+    @property
+    def del_labels_dir(self) -> str:
+        return path.join(self.del_dir, LABELS_DIR)
 
     @property
     def corrupted_dir(self) -> str:
@@ -85,11 +94,11 @@ class Config(BaseModel):
 
     @property
     def corrupted_images_dir(self) -> str:
-        return path.join(self.corrupted_dir, self.dirs.images)
+        return path.join(self.corrupted_dir, IMAGES_DIR)
 
     @property
     def corrupted_labels_dir(self) -> str:
-        return path.join(self.corrupted_dir, self.dirs.labels)
+        return path.join(self.corrupted_dir, LABELS_DIR)
 
     @property
     def sample_dir(self) -> str:
@@ -130,6 +139,9 @@ checkpoint_date_format = '%Y-%m-%d %H:%M:%S'
 
 CONFIG_FILE = 'config.yaml'
 
+IMAGES_DIR = 'images'
+LABELS_DIR = 'labels'
+
 JPG_EXT = '.jpg'
 TXT_EXT = '.txt'
 

src/dataset-visualiser.py

@@ -33,8 +33,8 @@ def visualise_dataset():
 def visualise_processed_folder():
 
     def show_image(img):
-        image_path = os.path.join(constants.config.processed_images_dir, img)
-        labels_path = os.path.join(constants.config.processed_labels_dir, f'{Path(img).stem}.txt')
+        image_path = os.path.join(constants.config.images_dir, img)
+        labels_path = os.path.join(constants.config.labels_dir, f'{Path(img).stem}.txt')
         img = ImageLabel(
             image_path=image_path,
             image=cv2.imread(image_path),
@@ -42,7 +42,7 @@ def visualise_processed_folder():
             labels=read_labels(labels_path)
         )
         img.visualize(annotation_classes)
-    images = os.listdir(constants.config.processed_images_dir)
+    images = os.listdir(constants.config.images_dir)
     cur = 0
     show_image(images[cur])
     pass

src/exports.py

@@ -62,7 +62,7 @@ def export_tensorrt(model_path):
 
 def form_data_sample(destination_path, size=500, write_txt_log=False):
     images = []
-    with scandir(constants.config.processed_images_dir) as imd:
+    with scandir(constants.config.images_dir) as imd:
         for image_file in imd:
             if not image_file.is_file():
                 continue

src/train.py

@@ -38,7 +38,7 @@ def form_dataset():
     shutil.rmtree(today_dataset, ignore_errors=True)
     makedirs(today_dataset)
     images = []
-    with scandir(constants.config.processed_images_dir) as imd:
+    with scandir(constants.config.images_dir) as imd:
         for image_file in imd:
             if not image_file.is_file():
                 continue
@@ -60,14 +60,20 @@ def copy_annotations(images, folder):
     global total_files_copied
     total_files_copied = 0
 
+    def _link_or_copy(src, dst):
+        try:
+            os.link(src, dst)
+        except OSError:
+            shutil.copy(src, dst)
+
     def copy_image(image):
         global total_files_copied
         total_files_copied += 1
         label_name = f'{Path(image.path).stem}.txt'
-        label_path = path.join(constants.config.processed_labels_dir, label_name)
+        label_path = path.join(constants.config.labels_dir, label_name)
         if check_label(label_path):
-            shutil.copy(image.path, path.join(destination_images, image.name))
-            shutil.copy(label_path, path.join(destination_labels, label_name))
+            _link_or_copy(image.path, path.join(destination_images, image.name))
+            _link_or_copy(label_path, path.join(destination_labels, label_name))
         else:
             shutil.copy(image.path, path.join(constants.config.corrupted_images_dir, image.name))
             shutil.copy(label_path, path.join(constants.config.corrupted_labels_dir, label_name))