Update configuration and test structure for improved clarity and functionality
- Modified `.gitignore` to include test fixture data while excluding test results. - Updated `config.yaml` to change the model from 'yolo11m.yaml' to 'yolo26m.pt'. - Enhanced `.cursor/rules/coderule.mdc` with additional guidelines for test environment consistency and infrastructure handling. - Revised autopilot state management in `_docs/_autopilot_state.md` to reflect current progress and tasks. - Removed outdated augmentation tests and adjusted dataset formation tests to align with the new structure. These changes streamline the configuration and testing processes, ensuring better organization and clarity in the project.
@@ -4,8 +4,10 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
_DATASET_IMAGES = _PROJECT_ROOT / "_docs/00_problem/input_data/dataset/images"
|
||||
_DATASET_LABELS = _PROJECT_ROOT / "_docs/00_problem/input_data/dataset/labels"
|
||||
_TESTS_DIR = Path(__file__).resolve().parent
|
||||
_TEST_ROOT = _TESTS_DIR / "root"
|
||||
_DATASET_IMAGES = _TEST_ROOT / "data" / "images"
|
||||
_DATASET_LABELS = _TEST_ROOT / "data" / "labels"
|
||||
_ONNX_MODEL = _PROJECT_ROOT / "_docs/00_problem/input_data/azaion.onnx"
|
||||
_CLASSES_JSON = _PROJECT_ROOT / "src" / "classes.json"
|
||||
_CONFIG_TEST = _PROJECT_ROOT / "config.test.yaml"
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
import concurrent.futures
|
||||
import random
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.conftest import apply_constants_patch
|
||||
|
||||
|
||||
def _patch_augmentation_paths(monkeypatch, base: Path):
|
||||
apply_constants_patch(monkeypatch, base)
|
||||
|
||||
|
||||
def _augment_annotation_with_total(monkeypatch):
|
||||
import augmentation as aug
|
||||
|
||||
orig = aug.Augmentator.augment_annotation
|
||||
|
||||
def wrapped(self, image_file):
|
||||
self.total_to_process = self.total_images_to_process
|
||||
return orig(self, image_file)
|
||||
|
||||
monkeypatch.setattr(aug.Augmentator, "augment_annotation", wrapped)
|
||||
|
||||
|
||||
def _seed():
|
||||
random.seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
|
||||
@pytest.mark.performance
|
||||
def test_pt_aug_01_throughput_ten_images_sixty_seconds(
|
||||
tmp_path, monkeypatch, sample_images_labels
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
_seed()
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(10)
|
||||
for p in src_img.glob("*.jpg"):
|
||||
shutil.copy2(p, img_dir / p.name)
|
||||
for p in src_lbl.glob("*.txt"):
|
||||
shutil.copy2(p, lbl_dir / p.name)
|
||||
# Act
|
||||
t0 = time.perf_counter()
|
||||
Augmentator().augment_annotations()
|
||||
elapsed = time.perf_counter() - t0
|
||||
# Assert
|
||||
assert elapsed <= 60.0
|
||||
|
||||
|
||||
@pytest.mark.performance
|
||||
def test_pt_aug_02_parallel_at_least_one_point_five_x_faster(
|
||||
tmp_path, monkeypatch, sample_images_labels
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
_seed()
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
proc_dir = Path(c.config.processed_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(10)
|
||||
for p in src_img.glob("*.jpg"):
|
||||
shutil.copy2(p, img_dir / p.name)
|
||||
for p in src_lbl.glob("*.txt"):
|
||||
shutil.copy2(p, lbl_dir / p.name)
|
||||
Path(c.config.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
names = sorted(p.name for p in img_dir.glob("*.jpg"))
|
||||
|
||||
class _E:
|
||||
__slots__ = ("name",)
|
||||
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
entries = [_E(n) for n in names]
|
||||
|
||||
# Act
|
||||
aug_seq = Augmentator()
|
||||
aug_seq.total_images_to_process = len(entries)
|
||||
t0 = time.perf_counter()
|
||||
for e in entries:
|
||||
aug_seq.augment_annotation(e)
|
||||
seq_elapsed = time.perf_counter() - t0
|
||||
|
||||
shutil.rmtree(proc_dir)
|
||||
Path(c.config.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
aug_par = Augmentator()
|
||||
aug_par.total_images_to_process = len(entries)
|
||||
t0 = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor() as ex:
|
||||
list(ex.map(aug_par.augment_annotation, entries))
|
||||
par_elapsed = time.perf_counter() - t0
|
||||
|
||||
# Assert
|
||||
assert seq_elapsed >= par_elapsed * 1.5
|
||||
|
After Width: | Height: | Size: 710 KiB |
|
After Width: | Height: | Size: 781 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 788 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
@@ -0,0 +1 @@
|
||||
0 0.63516 0.76054 0.05480 0.09305
|
||||
@@ -0,0 +1 @@
|
||||
0 0.61853 0.77915 0.05973 0.09305
|
||||
@@ -0,0 +1 @@
|
||||
0 0.47200 0.78007 0.26215 0.42338
|
||||
@@ -0,0 +1,2 @@
|
||||
0 0.76062 0.31074 0.08738 0.13238
|
||||
0 0.79600 0.20352 0.04985 0.08424
|
||||
@@ -0,0 +1,6 @@
|
||||
6 0.52646 0.69638 0.04738 0.18161
|
||||
6 0.56554 0.69856 0.03077 0.16848
|
||||
6 0.59908 0.69311 0.03754 0.17263
|
||||
6 0.70000 0.74293 0.04738 0.17176
|
||||
6 0.77046 0.69638 0.03077 0.14441
|
||||
6 0.73538 0.70190 0.03077 0.14660
|
||||
@@ -0,0 +1,4 @@
|
||||
6 0.63569 0.46827 0.04185 0.13675
|
||||
6 0.68492 0.38403 0.04800 0.13019
|
||||
6 0.61569 0.36161 0.03877 0.10065
|
||||
6 0.56708 0.44147 0.04985 0.12253
|
||||
@@ -0,0 +1 @@
|
||||
3 0.74738 0.58588 0.33415 0.49450
|
||||
@@ -0,0 +1 @@
|
||||
0 0.91200 0.50492 0.09846 0.08971
|
||||
@@ -0,0 +1 @@
|
||||
0 0.87846 0.50930 0.09785 0.07658
|
||||
@@ -0,0 +1 @@
|
||||
2 0.53169 0.44475 0.06523 0.12691
|
||||
@@ -0,0 +1 @@
|
||||
2 0.51323 0.45679 0.08369 0.08752
|
||||
@@ -0,0 +1 @@
|
||||
0 0.90154 0.37309 0.08246 0.10831
|
||||
@@ -0,0 +1,4 @@
|
||||
6 0.35723 0.48851 0.03015 0.12253
|
||||
6 0.21415 0.44147 0.02954 0.11597
|
||||
6 0.24277 0.44147 0.02769 0.11597
|
||||
6 0.28892 0.44530 0.02769 0.08205
|
||||
@@ -0,0 +1 @@
|
||||
3 0.24338 0.33097 0.16308 0.21443
|
||||
@@ -0,0 +1 @@
|
||||
3 0.53200 0.18055 0.07815 0.08862
|
||||
@@ -0,0 +1 @@
|
||||
1 0.63538 0.20571 0.05354 0.11050
|
||||
@@ -0,0 +1 @@
|
||||
3 0.70092 0.15046 0.04431 0.06564
|
||||
@@ -0,0 +1 @@
|
||||
1 0.28431 0.81398 0.12677 0.33805
|
||||
@@ -0,0 +1,2 @@
|
||||
0 0.33108 0.75600 0.23262 0.35556
|
||||
6 0.10862 0.51641 0.03385 0.09080
|
||||
@@ -0,0 +1 @@
|
||||
0 0.41138 0.64113 0.29108 0.21552
|
||||
|
After Width: | Height: | Size: 710 KiB |
|
After Width: | Height: | Size: 781 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 1.5 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 788 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
@@ -0,0 +1 @@
|
||||
0 0.63516 0.76054 0.05480 0.09305
|
||||
@@ -0,0 +1 @@
|
||||
0 0.61853 0.77915 0.05973 0.09305
|
||||
@@ -0,0 +1 @@
|
||||
0 0.47200 0.78007 0.26215 0.42338
|
||||
@@ -0,0 +1,2 @@
|
||||
0 0.76062 0.31074 0.08738 0.13238
|
||||
0 0.79600 0.20352 0.04985 0.08424
|
||||
@@ -0,0 +1,6 @@
|
||||
6 0.52646 0.69638 0.04738 0.18161
|
||||
6 0.56554 0.69856 0.03077 0.16848
|
||||
6 0.59908 0.69311 0.03754 0.17263
|
||||
6 0.70000 0.74293 0.04738 0.17176
|
||||
6 0.77046 0.69638 0.03077 0.14441
|
||||
6 0.73538 0.70190 0.03077 0.14660
|
||||
@@ -0,0 +1,4 @@
|
||||
6 0.63569 0.46827 0.04185 0.13675
|
||||
6 0.68492 0.38403 0.04800 0.13019
|
||||
6 0.61569 0.36161 0.03877 0.10065
|
||||
6 0.56708 0.44147 0.04985 0.12253
|
||||
@@ -0,0 +1 @@
|
||||
3 0.74738 0.58588 0.33415 0.49450
|
||||
@@ -0,0 +1 @@
|
||||
0 0.91200 0.50492 0.09846 0.08971
|
||||
@@ -0,0 +1 @@
|
||||
0 0.87846 0.50930 0.09785 0.07658
|
||||
@@ -0,0 +1 @@
|
||||
2 0.53169 0.44475 0.06523 0.12691
|
||||
@@ -0,0 +1 @@
|
||||
2 0.51323 0.45679 0.08369 0.08752
|
||||
@@ -0,0 +1 @@
|
||||
0 0.90154 0.37309 0.08246 0.10831
|
||||
@@ -0,0 +1,4 @@
|
||||
6 0.35723 0.48851 0.03015 0.12253
|
||||
6 0.21415 0.44147 0.02954 0.11597
|
||||
6 0.24277 0.44147 0.02769 0.11597
|
||||
6 0.28892 0.44530 0.02769 0.08205
|
||||
@@ -0,0 +1 @@
|
||||
3 0.24338 0.33097 0.16308 0.21443
|
||||
@@ -0,0 +1 @@
|
||||
3 0.53200 0.18055 0.07815 0.08862
|
||||
@@ -0,0 +1 @@
|
||||
1 0.63538 0.20571 0.05354 0.11050
|
||||
@@ -0,0 +1 @@
|
||||
3 0.70092 0.15046 0.04431 0.06564
|
||||
@@ -0,0 +1 @@
|
||||
1 0.28431 0.81398 0.12677 0.33805
|
||||
@@ -0,0 +1,2 @@
|
||||
0 0.33108 0.75600 0.23262 0.35556
|
||||
6 0.10862 0.51641 0.03385 0.09080
|
||||
@@ -0,0 +1 @@
|
||||
0 0.41138 0.64113 0.29108 0.21552
|
||||
@@ -1,267 +0,0 @@
|
||||
import random
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from tests.conftest import apply_constants_patch
|
||||
|
||||
|
||||
def _patch_augmentation_paths(monkeypatch, base: Path):
|
||||
apply_constants_patch(monkeypatch, base)
|
||||
|
||||
|
||||
def _seed():
|
||||
random.seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
|
||||
def _augment_annotation_with_total(monkeypatch):
|
||||
import augmentation as aug
|
||||
|
||||
orig = aug.Augmentator.augment_annotation
|
||||
|
||||
def wrapped(self, image_file):
|
||||
self.total_to_process = self.total_images_to_process
|
||||
return orig(self, image_file)
|
||||
|
||||
monkeypatch.setattr(aug.Augmentator, "augment_annotation", wrapped)
|
||||
|
||||
|
||||
def test_bt_aug_01_augment_inner_returns_eight_image_labels(
|
||||
tmp_path, monkeypatch, fixture_images_dir, fixture_labels_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
|
||||
stem = sorted(fixture_images_dir.glob("*.jpg"))[0].stem
|
||||
img_path = fixture_images_dir / f"{stem}.jpg"
|
||||
lbl_path = fixture_labels_dir / f"{stem}.txt"
|
||||
img = cv2.imdecode(np.fromfile(str(img_path), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
aug = Augmentator()
|
||||
labels = aug.read_labels(lbl_path)
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=labels,
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
assert len(out) == 8
|
||||
|
||||
|
||||
def test_bt_aug_02_naming_convention(tmp_path, monkeypatch, fixture_images_dir, fixture_labels_dir):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
stem = "test_image"
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
src_img = sorted(fixture_images_dir.glob("*.jpg"))[0]
|
||||
img = cv2.imdecode(np.fromfile(str(src_img), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
lbl_path = fixture_labels_dir / f"{src_img.stem}.txt"
|
||||
labels = Augmentator().read_labels(lbl_path)
|
||||
aug = Augmentator()
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=labels,
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
names = [Path(o.image_path).name for o in out]
|
||||
expected = [f"{stem}.jpg"] + [f"{stem}_{i}.jpg" for i in range(1, 8)]
|
||||
assert names == expected
|
||||
lbl_names = [Path(o.labels_path).name for o in out]
|
||||
expected_lbl = [f"{stem}.txt"] + [f"{stem}_{i}.txt" for i in range(1, 8)]
|
||||
assert lbl_names == expected_lbl
|
||||
|
||||
|
||||
def _all_coords_in_unit(labels_list):
|
||||
for row in labels_list:
|
||||
for j in range(4):
|
||||
v = float(row[j])
|
||||
if v < 0.0 or v > 1.0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def test_bt_aug_03_all_bbox_coords_in_zero_one(
|
||||
tmp_path, monkeypatch, fixture_images_dir, fixture_labels_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
stem = sorted(fixture_images_dir.glob("*.jpg"))[0].stem
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
img_path = fixture_images_dir / f"{stem}.jpg"
|
||||
lbl_path = fixture_labels_dir / f"{stem}.txt"
|
||||
img = cv2.imdecode(np.fromfile(str(img_path), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
aug = Augmentator()
|
||||
labels = aug.read_labels(lbl_path)
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=labels,
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
for o in out:
|
||||
for row in o.labels:
|
||||
assert len(row) >= 5
|
||||
assert _all_coords_in_unit(o.labels)
|
||||
|
||||
|
||||
def test_bt_aug_04_correct_bboxes_clips_edge(tmp_path, monkeypatch):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
from augmentation import Augmentator
|
||||
|
||||
aug = Augmentator()
|
||||
m = aug.correct_margin
|
||||
inp = [[0.99, 0.5, 0.2, 0.1, 0]]
|
||||
# Act
|
||||
res = aug.correct_bboxes(inp)
|
||||
# Assert
|
||||
assert len(res) == 1
|
||||
x, y, w, h, _ = res[0]
|
||||
hw, hh = 0.5 * w, 0.5 * h
|
||||
assert x - hw >= m - 1e-9
|
||||
assert x + hw <= 1.0 - m + 1e-9
|
||||
assert y - hh >= m - 1e-9
|
||||
assert y + hh <= 1.0 - m + 1e-9
|
||||
|
||||
|
||||
def test_bt_aug_05_tiny_bbox_removed_after_clipping(tmp_path, monkeypatch):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
from augmentation import Augmentator
|
||||
|
||||
aug = Augmentator()
|
||||
inp = [[0.995, 0.5, 0.01, 0.5, 0]]
|
||||
# Act
|
||||
res = aug.correct_bboxes(inp)
|
||||
# Assert
|
||||
assert res == []
|
||||
|
||||
|
||||
def test_bt_aug_06_empty_label_eight_outputs_empty_labels(
|
||||
tmp_path, monkeypatch, fixture_images_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
stem = "empty_case"
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
src_img = sorted(fixture_images_dir.glob("*.jpg"))[0]
|
||||
img = cv2.imdecode(np.fromfile(str(src_img), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
aug = Augmentator()
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=[],
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
assert len(out) == 8
|
||||
for o in out:
|
||||
assert o.labels == []
|
||||
|
||||
|
||||
def test_bt_aug_07_full_pipeline_five_images_forty_outputs(
|
||||
tmp_path, monkeypatch, sample_images_labels
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
_seed()
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(5)
|
||||
for p in src_img.glob("*.jpg"):
|
||||
shutil.copy2(p, img_dir / p.name)
|
||||
for p in src_lbl.glob("*.txt"):
|
||||
shutil.copy2(p, lbl_dir / p.name)
|
||||
# Act
|
||||
Augmentator().augment_annotations()
|
||||
# Assert
|
||||
proc_img = Path(c.config.processed_images_dir)
|
||||
proc_lbl = Path(c.config.processed_labels_dir)
|
||||
assert len(list(proc_img.glob("*.jpg"))) == 40
|
||||
assert len(list(proc_lbl.glob("*.txt"))) == 40
|
||||
|
||||
|
||||
def test_bt_aug_08_skips_already_processed(tmp_path, monkeypatch, sample_images_labels):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
_seed()
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
proc_img = Path(c.config.processed_images_dir)
|
||||
proc_lbl = Path(c.config.processed_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
proc_img.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(5)
|
||||
jpgs = sorted(src_img.glob("*.jpg"))
|
||||
for p in jpgs:
|
||||
shutil.copy2(p, img_dir / p.name)
|
||||
for p in src_lbl.glob("*.txt"):
|
||||
shutil.copy2(p, lbl_dir / p.name)
|
||||
markers = []
|
||||
for p in jpgs[:3]:
|
||||
dst = proc_img / p.name
|
||||
shutil.copy2(p, dst)
|
||||
markers.append(dst.read_bytes())
|
||||
# Act
|
||||
Augmentator().augment_annotations()
|
||||
# Assert
|
||||
after_jpgs = list(proc_img.glob("*.jpg"))
|
||||
assert len(after_jpgs) == 19
|
||||
assert len(list(proc_lbl.glob("*.txt"))) == 16
|
||||
for i, p in enumerate(jpgs[:3]):
|
||||
assert (proc_img / p.name).read_bytes() == markers[i]
|
||||
@@ -1,143 +0,0 @@
|
||||
import random
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.conftest import apply_constants_patch
|
||||
|
||||
|
||||
def _patch_augmentation_paths(monkeypatch, base: Path):
|
||||
apply_constants_patch(monkeypatch, base)
|
||||
|
||||
|
||||
def _augment_annotation_with_total(monkeypatch):
|
||||
import augmentation as aug
|
||||
|
||||
orig = aug.Augmentator.augment_annotation
|
||||
|
||||
def wrapped(self, image_file):
|
||||
self.total_to_process = self.total_images_to_process
|
||||
return orig(self, image_file)
|
||||
|
||||
monkeypatch.setattr(aug.Augmentator, "augment_annotation", wrapped)
|
||||
|
||||
|
||||
def _seed():
|
||||
random.seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
|
||||
@pytest.mark.resilience
|
||||
def test_rt_aug_01_corrupted_image_skipped(
|
||||
tmp_path, monkeypatch, fixture_images_dir, fixture_labels_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
_seed()
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = sorted(fixture_images_dir.glob("*.jpg"))[0].stem
|
||||
shutil.copy2(fixture_images_dir / f"{stem}.jpg", img_dir / f"{stem}.jpg")
|
||||
shutil.copy2(fixture_labels_dir / f"{stem}.txt", lbl_dir / f"{stem}.txt")
|
||||
raw = (fixture_images_dir / f"{stem}.jpg").read_bytes()[:200]
|
||||
(img_dir / "corrupted_trunc.jpg").write_bytes(raw)
|
||||
# Act
|
||||
Augmentator().augment_annotations()
|
||||
# Assert
|
||||
proc_img = Path(c.config.processed_images_dir)
|
||||
assert len(list(proc_img.glob("*.jpg"))) == 8
|
||||
|
||||
|
||||
@pytest.mark.resilience
|
||||
def test_rt_aug_02_missing_label_no_crash(tmp_path, monkeypatch, fixture_images_dir):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_augment_annotation_with_total(monkeypatch)
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = "no_label_here"
|
||||
shutil.copy2(sorted(fixture_images_dir.glob("*.jpg"))[0], img_dir / f"{stem}.jpg")
|
||||
aug = Augmentator()
|
||||
aug.total_images_to_process = 1
|
||||
# Act
|
||||
aug.augment_annotation(SimpleNamespace(name=f"{stem}.jpg"))
|
||||
# Assert
|
||||
assert len(list(Path(c.config.processed_images_dir).glob("*.jpg"))) == 0
|
||||
|
||||
|
||||
@pytest.mark.resilience
|
||||
def test_rt_aug_03_narrow_bbox_fewer_or_eight_variants(
|
||||
tmp_path, monkeypatch, fixture_images_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
stem = "narrow_bbox"
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
src_img = sorted(fixture_images_dir.glob("*.jpg"))[0]
|
||||
img = cv2.imdecode(np.fromfile(str(src_img), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
aug = Augmentator()
|
||||
labels = [[0.5, 0.5, 0.0005, 0.0005, 0]]
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=labels,
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
assert 1 <= len(out) <= 8
|
||||
|
||||
|
||||
@pytest.mark.resource_limit
|
||||
def test_rl_aug_01_augment_inner_exactly_eight_outputs(
|
||||
tmp_path, monkeypatch, fixture_images_dir, fixture_labels_dir
|
||||
):
|
||||
# Arrange
|
||||
_patch_augmentation_paths(monkeypatch, tmp_path)
|
||||
_seed()
|
||||
from augmentation import Augmentator
|
||||
from dto.imageLabel import ImageLabel
|
||||
|
||||
stem = sorted(fixture_images_dir.glob("*.jpg"))[0].stem
|
||||
img_path = fixture_images_dir / f"{stem}.jpg"
|
||||
lbl_path = fixture_labels_dir / f"{stem}.txt"
|
||||
img = cv2.imdecode(np.fromfile(str(img_path), dtype=np.uint8), cv2.IMREAD_COLOR)
|
||||
aug = Augmentator()
|
||||
labels = aug.read_labels(lbl_path)
|
||||
proc_img = Path(tmp_path) / "azaion" / "data-processed" / "images" / f"{stem}.jpg"
|
||||
proc_lbl = Path(tmp_path) / "azaion" / "data-processed" / "labels" / f"{stem}.txt"
|
||||
proc_img.parent.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.parent.mkdir(parents=True, exist_ok=True)
|
||||
img_ann = ImageLabel(
|
||||
image_path=str(proc_img),
|
||||
image=img,
|
||||
labels_path=str(proc_lbl),
|
||||
labels=labels,
|
||||
)
|
||||
# Act
|
||||
out = aug.augment_inner(img_ann)
|
||||
# Assert
|
||||
assert len(out) == 8
|
||||
@@ -55,15 +55,15 @@ def test_bt_dsf_01_split_ratio_70_20_10(
|
||||
constants_patch,
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
100,
|
||||
20,
|
||||
set(),
|
||||
)
|
||||
# Act
|
||||
train.form_dataset()
|
||||
# Assert
|
||||
assert _count_jpg(Path(today_ds, "train", "images")) == 70
|
||||
assert _count_jpg(Path(today_ds, "valid", "images")) == 20
|
||||
assert _count_jpg(Path(today_ds, "test", "images")) == 10
|
||||
assert _count_jpg(Path(today_ds, "train", "images")) == 14
|
||||
assert _count_jpg(Path(today_ds, "valid", "images")) == 4
|
||||
assert _count_jpg(Path(today_ds, "test", "images")) == 2
|
||||
|
||||
|
||||
def test_bt_dsf_02_six_subdirectories(
|
||||
@@ -80,7 +80,7 @@ def test_bt_dsf_02_six_subdirectories(
|
||||
constants_patch,
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
100,
|
||||
20,
|
||||
set(),
|
||||
)
|
||||
# Act
|
||||
@@ -95,7 +95,7 @@ def test_bt_dsf_02_six_subdirectories(
|
||||
assert (base / "test" / "labels").is_dir()
|
||||
|
||||
|
||||
def test_bt_dsf_03_total_files_one_hundred(
|
||||
def test_bt_dsf_03_total_files_twenty(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
constants_patch,
|
||||
@@ -109,7 +109,7 @@ def test_bt_dsf_03_total_files_one_hundred(
|
||||
constants_patch,
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
100,
|
||||
20,
|
||||
set(),
|
||||
)
|
||||
# Act
|
||||
@@ -120,7 +120,7 @@ def test_bt_dsf_03_total_files_one_hundred(
|
||||
+ _count_jpg(Path(today_ds, "valid", "images"))
|
||||
+ _count_jpg(Path(today_ds, "test", "images"))
|
||||
)
|
||||
assert n == 100
|
||||
assert n == 20
|
||||
|
||||
|
||||
def test_bt_dsf_04_corrupted_labels_quarantined(
|
||||
@@ -131,7 +131,7 @@ def test_bt_dsf_04_corrupted_labels_quarantined(
|
||||
fixture_labels_dir,
|
||||
):
|
||||
# Arrange
|
||||
stems = [p.stem for p in sorted(fixture_images_dir.glob("*.jpg"))[:100]]
|
||||
stems = [p.stem for p in sorted(fixture_images_dir.glob("*.jpg"))[:20]]
|
||||
corrupt = set(stems[:5])
|
||||
train, today_ds = _prepare_form_dataset(
|
||||
monkeypatch,
|
||||
@@ -139,7 +139,7 @@ def test_bt_dsf_04_corrupted_labels_quarantined(
|
||||
constants_patch,
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
100,
|
||||
20,
|
||||
corrupt,
|
||||
)
|
||||
# Act
|
||||
@@ -150,7 +150,7 @@ def test_bt_dsf_04_corrupted_labels_quarantined(
|
||||
+ _count_jpg(Path(today_ds, "valid", "images"))
|
||||
+ _count_jpg(Path(today_ds, "test", "images"))
|
||||
)
|
||||
assert split_total == 95
|
||||
assert split_total == 15
|
||||
assert _count_jpg(c_mod.config.corrupted_images_dir) == 5
|
||||
assert len(list(Path(c_mod.config.corrupted_labels_dir).glob("*.txt"))) == 5
|
||||
|
||||
@@ -202,7 +202,7 @@ def test_rl_dsf_02_no_filename_duplication_across_splits(
|
||||
constants_patch,
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
100,
|
||||
20,
|
||||
set(),
|
||||
)
|
||||
# Act
|
||||
@@ -214,4 +214,4 @@ def test_rl_dsf_02_no_filename_duplication_across_splits(
|
||||
for f in (base / split / "images").glob("*.jpg"):
|
||||
names.append(f.name)
|
||||
assert len(names) == len(set(names))
|
||||
assert len(names) == 100
|
||||
assert len(names) == 20
|
||||
|
||||
@@ -3,12 +3,12 @@ import constants as c
|
||||
|
||||
def test_fixture_images_dir_has_jpegs(fixture_images_dir):
|
||||
jpgs = list(fixture_images_dir.glob("*.jpg"))
|
||||
assert len(jpgs) == 100
|
||||
assert len(jpgs) == 20
|
||||
|
||||
|
||||
def test_fixture_labels_dir_has_yolo_labels(fixture_labels_dir):
|
||||
txts = list(fixture_labels_dir.glob("*.txt"))
|
||||
assert len(txts) == 100
|
||||
assert len(txts) == 20
|
||||
|
||||
|
||||
def test_fixture_onnx_model_bytes(fixture_onnx_model):
|
||||
@@ -54,6 +54,6 @@ def test_empty_label_file(empty_label):
|
||||
|
||||
def test_constants_patch_uses_tmp(constants_patch, tmp_path):
|
||||
constants_patch(tmp_path)
|
||||
assert c.config.azaion.startswith(str(tmp_path))
|
||||
assert c.config.root.startswith(str(tmp_path))
|
||||
assert c.config.data_dir.startswith(str(tmp_path))
|
||||
assert c.config.current_onnx_model.startswith(str(tmp_path))
|
||||
|
||||
@@ -9,33 +9,18 @@ import constants as c
|
||||
import train as train_mod
|
||||
import exports as exports_mod
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
_DATASET_IMAGES = _PROJECT_ROOT / "_docs/00_problem/input_data/dataset/images"
|
||||
_DATASET_LABELS = _PROJECT_ROOT / "_docs/00_problem/input_data/dataset/labels"
|
||||
_CONFIG_TEST = _PROJECT_ROOT / "config.test.yaml"
|
||||
_TESTS_DIR = Path(__file__).resolve().parent
|
||||
_TEST_ROOT = _TESTS_DIR / "root"
|
||||
_DATASET_IMAGES = _TEST_ROOT / "data" / "images"
|
||||
_CONFIG_TEST = _TESTS_DIR.parent / "config.test.yaml"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def e2e_result(tmp_path_factory):
|
||||
base = tmp_path_factory.mktemp("e2e")
|
||||
|
||||
def e2e_result():
|
||||
old_config = c.config
|
||||
c.config = c.Config.from_yaml(str(_CONFIG_TEST), root=str(base / "azaion"))
|
||||
c.config = c.Config.from_yaml(str(_CONFIG_TEST), root=str(_TEST_ROOT))
|
||||
|
||||
data_img = Path(c.config.data_images_dir)
|
||||
data_lbl = Path(c.config.data_labels_dir)
|
||||
data_img.mkdir(parents=True)
|
||||
data_lbl.mkdir(parents=True)
|
||||
Path(c.config.models_dir).mkdir(parents=True)
|
||||
|
||||
for img in sorted(_DATASET_IMAGES.glob("*.jpg")):
|
||||
shutil.copy2(img, data_img / img.name)
|
||||
lbl = _DATASET_LABELS / f"{img.stem}.txt"
|
||||
if lbl.exists():
|
||||
shutil.copy2(lbl, data_lbl / lbl.name)
|
||||
|
||||
from augmentation import Augmentator
|
||||
Augmentator().augment_annotations()
|
||||
Path(c.config.models_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
train_mod.train_dataset()
|
||||
|
||||
@@ -48,15 +33,14 @@ def e2e_result(tmp_path_factory):
|
||||
"today_dataset": today_ds,
|
||||
}
|
||||
|
||||
shutil.rmtree(c.config.datasets_dir, ignore_errors=True)
|
||||
shutil.rmtree(c.config.models_dir, ignore_errors=True)
|
||||
shutil.rmtree(c.config.corrupted_dir, ignore_errors=True)
|
||||
c.config = old_config
|
||||
|
||||
|
||||
@pytest.mark.e2e
|
||||
class TestTrainingPipeline:
|
||||
def test_augmentation_produced_output(self, e2e_result):
|
||||
proc = Path(c.config.processed_images_dir)
|
||||
assert len(list(proc.glob("*.jpg"))) == 800
|
||||
|
||||
def test_dataset_formed(self, e2e_result):
|
||||
base = Path(e2e_result["today_dataset"])
|
||||
for split in ("train", "valid", "test"):
|
||||
@@ -66,7 +50,7 @@ class TestTrainingPipeline:
|
||||
len(list((base / s / "images").glob("*.jpg")))
|
||||
for s in ("train", "valid", "test")
|
||||
)
|
||||
assert total == 800
|
||||
assert total == 20
|
||||
|
||||
def test_data_yaml_created(self, e2e_result):
|
||||
yaml_path = Path(e2e_result["today_dataset"]) / "data.yaml"
|
||||
|
||||