diff --git a/.gitignore b/.gitignore index 717cf1f..d80d5c5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,3 @@ venv # Test results tests/test-results/ - -# Test fixture data — override global ignores for committed test data -!tests/root/ -!tests/root/** \ No newline at end of file diff --git a/_docs/_autopilot_state.md b/_docs/_autopilot_state.md index 87b3c26..3795bad 100644 --- a/_docs/_autopilot_state.md +++ b/_docs/_autopilot_state.md @@ -5,5 +5,5 @@ flow: existing-code step: 7 name: Refactor status: in_progress -sub_step: 4 — Execution (Batch 1 done: AZ-165, AZ-166, AZ-167; next: Batch 2 AZ-168) +sub_step: 4 — Execution (All batches done: AZ-165, AZ-166, AZ-167, AZ-168, AZ-169; pending: final test run, commit) retry_count: 0 diff --git a/config.test.yaml b/config.test.yaml index 8d8f43a..3851947 100644 --- a/config.test.yaml +++ b/config.test.yaml @@ -1,7 +1,7 @@ training: model: 'yolo11n.yaml' epochs: 1 - batch: 4 + batch: 64 imgsz: 320 save_period: 1 workers: 0 diff --git a/config.yaml b/config.yaml index b7fff65..10da702 100644 --- a/config.yaml +++ b/config.yaml @@ -14,10 +14,8 @@ dirs: root: '/azaion' data: 'data' data_seed: 'data-seed' - data_processed: 'data-processed' + data_deleted: 'data_deleted' - images: 'images' - labels: 'labels' training: model: 'yolo26m.pt' diff --git a/src/annotation-queue/annotation_queue_handler.py b/src/annotation-queue/annotation_queue_handler.py index 543ffc4..3f85572 100644 --- a/src/annotation-queue/annotation_queue_handler.py +++ b/src/annotation-queue/annotation_queue_handler.py @@ -40,36 +40,26 @@ class AnnotationQueueHandler: class AnnotationName: def __init__(self, h, name): - self.img_data = path.join(h.data_dir, h.images_dir, f"{name}{h.JPG_EXT}") - self.lbl_data = path.join(h.data_dir, h.labels_dir, f"{name}{h.TXT_EXT}") - self.img_seed = path.join(h.data_seed_dir, h.images_dir, f"{name}{h.JPG_EXT}") - self.lbl_seed = path.join(h.data_seed_dir, h.labels_dir, f"{name}{h.TXT_EXT}") + self.img_data = path.join(h.cfg.images_dir, f"{name}{h.JPG_EXT}") + self.lbl_data = path.join(h.cfg.labels_dir, f"{name}{h.TXT_EXT}") + self.img_seed = path.join(h.cfg.seed_images_dir, f"{name}{h.JPG_EXT}") + self.lbl_seed = path.join(h.cfg.seed_labels_dir, f"{name}{h.TXT_EXT}") def __init__(self): - cfg = constants.config - self.data_dir = path.join(cfg.dirs.root, cfg.dirs.data) - self.data_seed_dir = path.join(cfg.dirs.root, cfg.dirs.data_seed) - self.images_dir = cfg.dirs.images - self.labels_dir = cfg.dirs.labels + self.cfg = constants.config - self.del_img_dir = path.join(cfg.dirs.root, cfg.dirs.data_deleted, self.images_dir) - self.del_lbl_dir = path.join(cfg.dirs.root, cfg.dirs.data_deleted, self.labels_dir) - - makedirs(path.join(self.data_dir, self.images_dir), exist_ok=True) - makedirs(path.join(self.data_dir, self.labels_dir), exist_ok=True) - makedirs(path.join(self.data_seed_dir, self.images_dir), exist_ok=True) - makedirs(path.join(self.data_seed_dir, self.labels_dir), exist_ok=True) - - makedirs(self.del_img_dir, exist_ok=True) - makedirs(self.del_lbl_dir, exist_ok=True) + for d in (self.cfg.images_dir, self.cfg.labels_dir, + self.cfg.seed_images_dir, self.cfg.seed_labels_dir, + self.cfg.del_images_dir, self.cfg.del_labels_dir): + makedirs(d, exist_ok=True) self.consumer = Consumer( - host=cfg.queue.host, - port=cfg.queue.port, - username=cfg.queue.consumer_user, - password=cfg.queue.consumer_pw + host=self.cfg.queue.host, + port=self.cfg.queue.port, + username=self.cfg.queue.consumer_user, + password=self.cfg.queue.consumer_pw ) - self.queue_name = cfg.queue.name + self.queue_name = self.cfg.queue.name try: with open(self.OFFSET_FILE, 'r') as f: @@ -154,14 +144,14 @@ class AnnotationQueueHandler: for name in msg.annotation_names: a = self.AnnotationName(self, name) if path.exists(a.img_data): - shutil.move(a.img_data, self.del_img_dir) + shutil.move(a.img_data, self.cfg.del_images_dir) if path.exists(a.img_seed): - shutil.move(a.img_seed, self.del_img_dir) + shutil.move(a.img_seed, self.cfg.del_images_dir) if path.exists(a.lbl_data): - shutil.move(a.lbl_data, self.del_lbl_dir) + shutil.move(a.lbl_data, self.cfg.del_labels_dir) if path.exists(a.lbl_seed): - shutil.move(a.lbl_seed, self.del_lbl_dir) + shutil.move(a.lbl_seed, self.cfg.del_labels_dir) if __name__ == '__main__': diff --git a/src/augmentation.py b/src/augmentation.py index 1d5a1a3..cb030f5 100644 --- a/src/augmentation.py +++ b/src/augmentation.py @@ -95,8 +95,8 @@ class Augmentator: def augment_annotation(self, image_file): try: - image_path = os.path.join(constants.config.data_images_dir, image_file.name) - labels_path = os.path.join(constants.config.data_labels_dir, f'{Path(str(image_path)).stem}.txt') + image_path = os.path.join(constants.config.images_dir, image_file.name) + labels_path = os.path.join(constants.config.labels_dir, f'{Path(str(image_path)).stem}.txt') image = cv2.imdecode(np.fromfile(image_path, dtype=np.uint8), cv2.IMREAD_UNCHANGED) img_ann = ImageLabel( @@ -134,7 +134,7 @@ class Augmentator: processed_images = set(f.name for f in os.scandir(constants.config.processed_images_dir)) images = [] - with os.scandir(constants.config.data_images_dir) as imd: + with os.scandir(constants.config.images_dir) as imd: for image_file in imd: if image_file.is_file() and image_file.name not in processed_images: images.append(image_file) diff --git a/src/constants.py b/src/constants.py index e883be6..1a6b74c 100644 --- a/src/constants.py +++ b/src/constants.py @@ -24,10 +24,7 @@ class DirsConfig(BaseModel): root: str = '/azaion' data: str = 'data' data_seed: str = 'data-seed' - data_processed: str = 'data-processed' data_deleted: str = 'data_deleted' - images: str = 'images' - labels: str = 'labels' class TrainingConfig(BaseModel): @@ -60,24 +57,36 @@ class Config(BaseModel): return path.join(self.dirs.root, self.dirs.data) @property - def data_images_dir(self) -> str: - return path.join(self.data_dir, self.dirs.images) + def images_dir(self) -> str: + return path.join(self.data_dir, IMAGES_DIR) @property - def data_labels_dir(self) -> str: - return path.join(self.data_dir, self.dirs.labels) + def labels_dir(self) -> str: + return path.join(self.data_dir, LABELS_DIR) @property - def processed_dir(self) -> str: - return path.join(self.dirs.root, self.dirs.data_processed) + def seed_dir(self) -> str: + return path.join(self.dirs.root, self.dirs.data_seed) @property - def processed_images_dir(self) -> str: - return path.join(self.processed_dir, self.dirs.images) + def seed_images_dir(self) -> str: + return path.join(self.seed_dir, IMAGES_DIR) @property - def processed_labels_dir(self) -> str: - return path.join(self.processed_dir, self.dirs.labels) + def seed_labels_dir(self) -> str: + return path.join(self.seed_dir, LABELS_DIR) + + @property + def del_dir(self) -> str: + return path.join(self.dirs.root, self.dirs.data_deleted) + + @property + def del_images_dir(self) -> str: + return path.join(self.del_dir, IMAGES_DIR) + + @property + def del_labels_dir(self) -> str: + return path.join(self.del_dir, LABELS_DIR) @property def corrupted_dir(self) -> str: @@ -85,11 +94,11 @@ class Config(BaseModel): @property def corrupted_images_dir(self) -> str: - return path.join(self.corrupted_dir, self.dirs.images) + return path.join(self.corrupted_dir, IMAGES_DIR) @property def corrupted_labels_dir(self) -> str: - return path.join(self.corrupted_dir, self.dirs.labels) + return path.join(self.corrupted_dir, LABELS_DIR) @property def sample_dir(self) -> str: @@ -130,6 +139,9 @@ checkpoint_date_format = '%Y-%m-%d %H:%M:%S' CONFIG_FILE = 'config.yaml' +IMAGES_DIR = 'images' +LABELS_DIR = 'labels' + JPG_EXT = '.jpg' TXT_EXT = '.txt' diff --git a/src/dataset-visualiser.py b/src/dataset-visualiser.py index 2b03ee1..b98a36f 100644 --- a/src/dataset-visualiser.py +++ b/src/dataset-visualiser.py @@ -33,8 +33,8 @@ def visualise_dataset(): def visualise_processed_folder(): def show_image(img): - image_path = os.path.join(constants.config.processed_images_dir, img) - labels_path = os.path.join(constants.config.processed_labels_dir, f'{Path(img).stem}.txt') + image_path = os.path.join(constants.config.images_dir, img) + labels_path = os.path.join(constants.config.labels_dir, f'{Path(img).stem}.txt') img = ImageLabel( image_path=image_path, image=cv2.imread(image_path), @@ -42,7 +42,7 @@ def visualise_processed_folder(): labels=read_labels(labels_path) ) img.visualize(annotation_classes) - images = os.listdir(constants.config.processed_images_dir) + images = os.listdir(constants.config.images_dir) cur = 0 show_image(images[cur]) pass diff --git a/src/exports.py b/src/exports.py index 5eb0c70..e8eb678 100644 --- a/src/exports.py +++ b/src/exports.py @@ -62,7 +62,7 @@ def export_tensorrt(model_path): def form_data_sample(destination_path, size=500, write_txt_log=False): images = [] - with scandir(constants.config.processed_images_dir) as imd: + with scandir(constants.config.images_dir) as imd: for image_file in imd: if not image_file.is_file(): continue diff --git a/src/train.py b/src/train.py index 8c0dc4a..0eb8ab5 100644 --- a/src/train.py +++ b/src/train.py @@ -38,7 +38,7 @@ def form_dataset(): shutil.rmtree(today_dataset, ignore_errors=True) makedirs(today_dataset) images = [] - with scandir(constants.config.processed_images_dir) as imd: + with scandir(constants.config.images_dir) as imd: for image_file in imd: if not image_file.is_file(): continue @@ -60,14 +60,20 @@ def copy_annotations(images, folder): global total_files_copied total_files_copied = 0 + def _link_or_copy(src, dst): + try: + os.link(src, dst) + except OSError: + shutil.copy(src, dst) + def copy_image(image): global total_files_copied total_files_copied += 1 label_name = f'{Path(image.path).stem}.txt' - label_path = path.join(constants.config.processed_labels_dir, label_name) + label_path = path.join(constants.config.labels_dir, label_name) if check_label(label_path): - shutil.copy(image.path, path.join(destination_images, image.name)) - shutil.copy(label_path, path.join(destination_labels, label_name)) + _link_or_copy(image.path, path.join(destination_images, image.name)) + _link_or_copy(label_path, path.join(destination_labels, label_name)) else: shutil.copy(image.path, path.join(constants.config.corrupted_images_dir, image.name)) shutil.copy(label_path, path.join(constants.config.corrupted_labels_dir, label_name)) diff --git a/tests/conftest.py b/tests/conftest.py index 45452b5..7d262b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,4 @@ +import csv import shutil from pathlib import Path @@ -14,6 +15,34 @@ _CONFIG_TEST = _PROJECT_ROOT / "config.test.yaml" collect_ignore = ["security_test.py", "imagelabel_visualize_test.py"] +_test_results = [] + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if report.when == "call" or (report.when == "setup" and report.skipped): + _test_results.append({ + "module": item.nodeid.rsplit("::", 1)[0], + "name": item.name, + "result": report.outcome.upper(), + "duration": round(report.duration, 3), + }) + + +def pytest_sessionfinish(session, exitstatus): + if not _test_results: + return + results_dir = Path(__file__).resolve().parent / "test-results" + results_dir.mkdir(exist_ok=True) + + with open(results_dir / "test-results.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["module", "test", "result", "duration_s"]) + for r in _test_results: + writer.writerow([r["module"], r["name"], r["result"], f"{r['duration']:.3f}"]) + def apply_constants_patch(monkeypatch, base: Path): import constants as c diff --git a/tests/performance/test_dataset_perf.py b/tests/performance/test_dataset_perf.py index 31c5d62..a098e46 100644 --- a/tests/performance/test_dataset_perf.py +++ b/tests/performance/test_dataset_perf.py @@ -20,16 +20,16 @@ def _prepare_form_dataset( constants_patch(tmp_path) import train - proc_img = Path(c_mod.config.processed_images_dir) - proc_lbl = Path(c_mod.config.processed_labels_dir) - proc_img.mkdir(parents=True, exist_ok=True) - proc_lbl.mkdir(parents=True, exist_ok=True) + data_img = Path(c_mod.config.images_dir) + data_lbl = Path(c_mod.config.labels_dir) + data_img.mkdir(parents=True, exist_ok=True) + data_lbl.mkdir(parents=True, exist_ok=True) imgs = sorted(fixture_images_dir.glob("*.jpg"))[:count] for p in imgs: stem = p.stem - shutil.copy2(fixture_images_dir / f"{stem}.jpg", proc_img / f"{stem}.jpg") - dst = proc_lbl / f"{stem}.txt" + shutil.copy2(fixture_images_dir / f"{stem}.jpg", data_img / f"{stem}.jpg") + dst = data_lbl / f"{stem}.txt" shutil.copy2(fixture_labels_dir / f"{stem}.txt", dst) if stem in corrupt_stems: dst.write_text("0 1.5 0.5 0.1 0.1\n", encoding="utf-8") diff --git a/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.jpg b/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.jpg deleted file mode 100644 index 87bf488..0000000 Binary files a/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.jpg b/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.jpg deleted file mode 100644 index 732e64d..0000000 Binary files a/tests/root/data-processed/images/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000106.jpg b/tests/root/data-processed/images/#Азов_з_неба_000106.jpg deleted file mode 100644 index f316b2b..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000106.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000113.jpg b/tests/root/data-processed/images/#Азов_з_неба_000113.jpg deleted file mode 100644 index 3871e98..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000113.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000148.jpg b/tests/root/data-processed/images/#Азов_з_неба_000148.jpg deleted file mode 100644 index 8c54eba..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000148.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000162.jpg b/tests/root/data-processed/images/#Азов_з_неба_000162.jpg deleted file mode 100644 index 480b9ac..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000162.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000173.jpg b/tests/root/data-processed/images/#Азов_з_неба_000173.jpg deleted file mode 100644 index ee949db..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000173.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000189.jpg b/tests/root/data-processed/images/#Азов_з_неба_000189.jpg deleted file mode 100644 index d9e1058..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000189.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000210.jpg b/tests/root/data-processed/images/#Азов_з_неба_000210.jpg deleted file mode 100644 index 85de419..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000210.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000245.jpg b/tests/root/data-processed/images/#Азов_з_неба_000245.jpg deleted file mode 100644 index 664dfc3..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000245.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000269.jpg b/tests/root/data-processed/images/#Азов_з_неба_000269.jpg deleted file mode 100644 index b765680..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000269.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000307.jpg b/tests/root/data-processed/images/#Азов_з_неба_000307.jpg deleted file mode 100644 index 1ad21e2..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000307.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000314.jpg b/tests/root/data-processed/images/#Азов_з_неба_000314.jpg deleted file mode 100644 index 89ecc54..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000314.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000320.jpg b/tests/root/data-processed/images/#Азов_з_неба_000320.jpg deleted file mode 100644 index dd66c48..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000320.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000338.jpg b/tests/root/data-processed/images/#Азов_з_неба_000338.jpg deleted file mode 100644 index 14744f5..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000338.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000346.jpg b/tests/root/data-processed/images/#Азов_з_неба_000346.jpg deleted file mode 100644 index 52986da..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000346.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000347.jpg b/tests/root/data-processed/images/#Азов_з_неба_000347.jpg deleted file mode 100644 index dcedfe7..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000347.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000373.jpg b/tests/root/data-processed/images/#Азов_з_неба_000373.jpg deleted file mode 100644 index 49ac20a..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000373.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000380.jpg b/tests/root/data-processed/images/#Азов_з_неба_000380.jpg deleted file mode 100644 index 27dd8d8..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000380.jpg and /dev/null differ diff --git a/tests/root/data-processed/images/#Азов_з_неба_000403.jpg b/tests/root/data-processed/images/#Азов_з_неба_000403.jpg deleted file mode 100644 index 4774f3c..0000000 Binary files a/tests/root/data-processed/images/#Азов_з_неба_000403.jpg and /dev/null differ diff --git a/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.txt b/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.txt deleted file mode 100644 index f0231e6..0000000 --- a/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001226.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.63516 0.76054 0.05480 0.09305 \ No newline at end of file diff --git a/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.txt b/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.txt deleted file mode 100644 index e03dfe2..0000000 --- a/tests/root/data-processed/labels/!Працюєпершийсамохіднийартилерійськийдивізіон.К_001278.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.61853 0.77915 0.05973 0.09305 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000106.txt b/tests/root/data-processed/labels/#Азов_з_неба_000106.txt deleted file mode 100644 index 304e549..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000106.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.47200 0.78007 0.26215 0.42338 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000113.txt b/tests/root/data-processed/labels/#Азов_з_неба_000113.txt deleted file mode 100644 index 104c11e..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000113.txt +++ /dev/null @@ -1,2 +0,0 @@ -0 0.76062 0.31074 0.08738 0.13238 -0 0.79600 0.20352 0.04985 0.08424 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000148.txt b/tests/root/data-processed/labels/#Азов_з_неба_000148.txt deleted file mode 100644 index 67e97f5..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000148.txt +++ /dev/null @@ -1,6 +0,0 @@ -6 0.52646 0.69638 0.04738 0.18161 -6 0.56554 0.69856 0.03077 0.16848 -6 0.59908 0.69311 0.03754 0.17263 -6 0.70000 0.74293 0.04738 0.17176 -6 0.77046 0.69638 0.03077 0.14441 -6 0.73538 0.70190 0.03077 0.14660 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000162.txt b/tests/root/data-processed/labels/#Азов_з_неба_000162.txt deleted file mode 100644 index 2216408..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000162.txt +++ /dev/null @@ -1,4 +0,0 @@ -6 0.63569 0.46827 0.04185 0.13675 -6 0.68492 0.38403 0.04800 0.13019 -6 0.61569 0.36161 0.03877 0.10065 -6 0.56708 0.44147 0.04985 0.12253 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000173.txt b/tests/root/data-processed/labels/#Азов_з_неба_000173.txt deleted file mode 100644 index 08ac396..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000173.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.74738 0.58588 0.33415 0.49450 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000189.txt b/tests/root/data-processed/labels/#Азов_з_неба_000189.txt deleted file mode 100644 index 546bd29..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000189.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.91200 0.50492 0.09846 0.08971 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000210.txt b/tests/root/data-processed/labels/#Азов_з_неба_000210.txt deleted file mode 100644 index 5a5eefa..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000210.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.87846 0.50930 0.09785 0.07658 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000245.txt b/tests/root/data-processed/labels/#Азов_з_неба_000245.txt deleted file mode 100644 index 7b9d776..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000245.txt +++ /dev/null @@ -1 +0,0 @@ -2 0.53169 0.44475 0.06523 0.12691 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000269.txt b/tests/root/data-processed/labels/#Азов_з_неба_000269.txt deleted file mode 100644 index a6dd311..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000269.txt +++ /dev/null @@ -1 +0,0 @@ -2 0.51323 0.45679 0.08369 0.08752 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000307.txt b/tests/root/data-processed/labels/#Азов_з_неба_000307.txt deleted file mode 100644 index f6614c0..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000307.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.90154 0.37309 0.08246 0.10831 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000314.txt b/tests/root/data-processed/labels/#Азов_з_неба_000314.txt deleted file mode 100644 index dfe2158..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000314.txt +++ /dev/null @@ -1,4 +0,0 @@ -6 0.35723 0.48851 0.03015 0.12253 -6 0.21415 0.44147 0.02954 0.11597 -6 0.24277 0.44147 0.02769 0.11597 -6 0.28892 0.44530 0.02769 0.08205 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000320.txt b/tests/root/data-processed/labels/#Азов_з_неба_000320.txt deleted file mode 100644 index b9f65f6..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000320.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.24338 0.33097 0.16308 0.21443 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000338.txt b/tests/root/data-processed/labels/#Азов_з_неба_000338.txt deleted file mode 100644 index f15f033..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000338.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.53200 0.18055 0.07815 0.08862 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000346.txt b/tests/root/data-processed/labels/#Азов_з_неба_000346.txt deleted file mode 100644 index cfb31fd..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000346.txt +++ /dev/null @@ -1 +0,0 @@ -1 0.63538 0.20571 0.05354 0.11050 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000347.txt b/tests/root/data-processed/labels/#Азов_з_неба_000347.txt deleted file mode 100644 index bccb5e5..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000347.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.70092 0.15046 0.04431 0.06564 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000373.txt b/tests/root/data-processed/labels/#Азов_з_неба_000373.txt deleted file mode 100644 index 5415a80..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000373.txt +++ /dev/null @@ -1 +0,0 @@ -1 0.28431 0.81398 0.12677 0.33805 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000380.txt b/tests/root/data-processed/labels/#Азов_з_неба_000380.txt deleted file mode 100644 index 194c6ac..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000380.txt +++ /dev/null @@ -1,2 +0,0 @@ -0 0.33108 0.75600 0.23262 0.35556 -6 0.10862 0.51641 0.03385 0.09080 \ No newline at end of file diff --git a/tests/root/data-processed/labels/#Азов_з_неба_000403.txt b/tests/root/data-processed/labels/#Азов_з_неба_000403.txt deleted file mode 100644 index 035b823..0000000 --- a/tests/root/data-processed/labels/#Азов_з_неба_000403.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.41138 0.64113 0.29108 0.21552 \ No newline at end of file diff --git a/tests/root/data/images/#Азов_з_неба_000106.jpg b/tests/root/data/images/#Азов_з_неба_000106.jpg deleted file mode 100644 index f316b2b..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000106.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000113.jpg b/tests/root/data/images/#Азов_з_неба_000113.jpg deleted file mode 100644 index 3871e98..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000113.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000148.jpg b/tests/root/data/images/#Азов_з_неба_000148.jpg deleted file mode 100644 index 8c54eba..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000148.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000162.jpg b/tests/root/data/images/#Азов_з_неба_000162.jpg deleted file mode 100644 index 480b9ac..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000162.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000173.jpg b/tests/root/data/images/#Азов_з_неба_000173.jpg deleted file mode 100644 index ee949db..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000173.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000189.jpg b/tests/root/data/images/#Азов_з_неба_000189.jpg deleted file mode 100644 index d9e1058..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000189.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000210.jpg b/tests/root/data/images/#Азов_з_неба_000210.jpg deleted file mode 100644 index 85de419..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000210.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000245.jpg b/tests/root/data/images/#Азов_з_неба_000245.jpg deleted file mode 100644 index 664dfc3..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000245.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000269.jpg b/tests/root/data/images/#Азов_з_неба_000269.jpg deleted file mode 100644 index b765680..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000269.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000307.jpg b/tests/root/data/images/#Азов_з_неба_000307.jpg deleted file mode 100644 index 1ad21e2..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000307.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000314.jpg b/tests/root/data/images/#Азов_з_неба_000314.jpg deleted file mode 100644 index 89ecc54..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000314.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000320.jpg b/tests/root/data/images/#Азов_з_неба_000320.jpg deleted file mode 100644 index dd66c48..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000320.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000338.jpg b/tests/root/data/images/#Азов_з_неба_000338.jpg deleted file mode 100644 index 14744f5..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000338.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000346.jpg b/tests/root/data/images/#Азов_з_неба_000346.jpg deleted file mode 100644 index 52986da..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000346.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000347.jpg b/tests/root/data/images/#Азов_з_неба_000347.jpg deleted file mode 100644 index dcedfe7..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000347.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000373.jpg b/tests/root/data/images/#Азов_з_неба_000373.jpg deleted file mode 100644 index 49ac20a..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000373.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000380.jpg b/tests/root/data/images/#Азов_з_неба_000380.jpg deleted file mode 100644 index 27dd8d8..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000380.jpg and /dev/null differ diff --git a/tests/root/data/images/#Азов_з_неба_000403.jpg b/tests/root/data/images/#Азов_з_неба_000403.jpg deleted file mode 100644 index 4774f3c..0000000 Binary files a/tests/root/data/images/#Азов_з_неба_000403.jpg and /dev/null differ diff --git a/tests/root/data/labels/#Азов_з_неба_000106.txt b/tests/root/data/labels/#Азов_з_неба_000106.txt deleted file mode 100644 index 304e549..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000106.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.47200 0.78007 0.26215 0.42338 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000113.txt b/tests/root/data/labels/#Азов_з_неба_000113.txt deleted file mode 100644 index 104c11e..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000113.txt +++ /dev/null @@ -1,2 +0,0 @@ -0 0.76062 0.31074 0.08738 0.13238 -0 0.79600 0.20352 0.04985 0.08424 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000148.txt b/tests/root/data/labels/#Азов_з_неба_000148.txt deleted file mode 100644 index 67e97f5..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000148.txt +++ /dev/null @@ -1,6 +0,0 @@ -6 0.52646 0.69638 0.04738 0.18161 -6 0.56554 0.69856 0.03077 0.16848 -6 0.59908 0.69311 0.03754 0.17263 -6 0.70000 0.74293 0.04738 0.17176 -6 0.77046 0.69638 0.03077 0.14441 -6 0.73538 0.70190 0.03077 0.14660 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000162.txt b/tests/root/data/labels/#Азов_з_неба_000162.txt deleted file mode 100644 index 2216408..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000162.txt +++ /dev/null @@ -1,4 +0,0 @@ -6 0.63569 0.46827 0.04185 0.13675 -6 0.68492 0.38403 0.04800 0.13019 -6 0.61569 0.36161 0.03877 0.10065 -6 0.56708 0.44147 0.04985 0.12253 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000173.txt b/tests/root/data/labels/#Азов_з_неба_000173.txt deleted file mode 100644 index 08ac396..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000173.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.74738 0.58588 0.33415 0.49450 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000189.txt b/tests/root/data/labels/#Азов_з_неба_000189.txt deleted file mode 100644 index 546bd29..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000189.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.91200 0.50492 0.09846 0.08971 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000210.txt b/tests/root/data/labels/#Азов_з_неба_000210.txt deleted file mode 100644 index 5a5eefa..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000210.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.87846 0.50930 0.09785 0.07658 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000245.txt b/tests/root/data/labels/#Азов_з_неба_000245.txt deleted file mode 100644 index 7b9d776..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000245.txt +++ /dev/null @@ -1 +0,0 @@ -2 0.53169 0.44475 0.06523 0.12691 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000269.txt b/tests/root/data/labels/#Азов_з_неба_000269.txt deleted file mode 100644 index a6dd311..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000269.txt +++ /dev/null @@ -1 +0,0 @@ -2 0.51323 0.45679 0.08369 0.08752 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000307.txt b/tests/root/data/labels/#Азов_з_неба_000307.txt deleted file mode 100644 index f6614c0..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000307.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.90154 0.37309 0.08246 0.10831 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000314.txt b/tests/root/data/labels/#Азов_з_неба_000314.txt deleted file mode 100644 index dfe2158..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000314.txt +++ /dev/null @@ -1,4 +0,0 @@ -6 0.35723 0.48851 0.03015 0.12253 -6 0.21415 0.44147 0.02954 0.11597 -6 0.24277 0.44147 0.02769 0.11597 -6 0.28892 0.44530 0.02769 0.08205 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000320.txt b/tests/root/data/labels/#Азов_з_неба_000320.txt deleted file mode 100644 index b9f65f6..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000320.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.24338 0.33097 0.16308 0.21443 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000338.txt b/tests/root/data/labels/#Азов_з_неба_000338.txt deleted file mode 100644 index f15f033..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000338.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.53200 0.18055 0.07815 0.08862 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000346.txt b/tests/root/data/labels/#Азов_з_неба_000346.txt deleted file mode 100644 index cfb31fd..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000346.txt +++ /dev/null @@ -1 +0,0 @@ -1 0.63538 0.20571 0.05354 0.11050 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000347.txt b/tests/root/data/labels/#Азов_з_неба_000347.txt deleted file mode 100644 index bccb5e5..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000347.txt +++ /dev/null @@ -1 +0,0 @@ -3 0.70092 0.15046 0.04431 0.06564 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000373.txt b/tests/root/data/labels/#Азов_з_неба_000373.txt deleted file mode 100644 index 5415a80..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000373.txt +++ /dev/null @@ -1 +0,0 @@ -1 0.28431 0.81398 0.12677 0.33805 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000380.txt b/tests/root/data/labels/#Азов_з_неба_000380.txt deleted file mode 100644 index 194c6ac..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000380.txt +++ /dev/null @@ -1,2 +0,0 @@ -0 0.33108 0.75600 0.23262 0.35556 -6 0.10862 0.51641 0.03385 0.09080 \ No newline at end of file diff --git a/tests/root/data/labels/#Азов_з_неба_000403.txt b/tests/root/data/labels/#Азов_з_неба_000403.txt deleted file mode 100644 index 035b823..0000000 --- a/tests/root/data/labels/#Азов_з_неба_000403.txt +++ /dev/null @@ -1 +0,0 @@ -0 0.41138 0.64113 0.29108 0.21552 \ No newline at end of file diff --git a/tests/test_dataset_formation.py b/tests/test_dataset_formation.py index 4f4ae20..7ee0e8f 100644 --- a/tests/test_dataset_formation.py +++ b/tests/test_dataset_formation.py @@ -19,16 +19,16 @@ def _prepare_form_dataset( constants_patch(tmp_path) import train - proc_img = Path(c_mod.config.processed_images_dir) - proc_lbl = Path(c_mod.config.processed_labels_dir) - proc_img.mkdir(parents=True, exist_ok=True) - proc_lbl.mkdir(parents=True, exist_ok=True) + data_img = Path(c_mod.config.images_dir) + data_lbl = Path(c_mod.config.labels_dir) + data_img.mkdir(parents=True, exist_ok=True) + data_lbl.mkdir(parents=True, exist_ok=True) imgs = sorted(fixture_images_dir.glob("*.jpg"))[:count] for p in imgs: stem = p.stem - shutil.copy2(fixture_images_dir / f"{stem}.jpg", proc_img / f"{stem}.jpg") - dst = proc_lbl / f"{stem}.txt" + shutil.copy2(fixture_images_dir / f"{stem}.jpg", data_img / f"{stem}.jpg") + dst = data_lbl / f"{stem}.txt" shutil.copy2(fixture_labels_dir / f"{stem}.txt", dst) if stem in corrupt_stems: dst.write_text("0 1.5 0.5 0.1 0.1\n", encoding="utf-8") @@ -156,7 +156,7 @@ def test_bt_dsf_04_corrupted_labels_quarantined( @pytest.mark.resilience -def test_rt_dsf_01_empty_processed_no_crash( +def test_rt_dsf_01_empty_data_no_crash( monkeypatch, tmp_path, constants_patch, diff --git a/tests/test_training_e2e.py b/tests/test_training_e2e.py index 21ba7e4..d3ba8a4 100644 --- a/tests/test_training_e2e.py +++ b/tests/test_training_e2e.py @@ -1,3 +1,4 @@ +import os import shutil from os import path from pathlib import Path @@ -13,15 +14,42 @@ _TESTS_DIR = Path(__file__).resolve().parent _TEST_ROOT = _TESTS_DIR / "root" _DATASET_IMAGES = _TEST_ROOT / "data" / "images" _CONFIG_TEST = _TESTS_DIR.parent / "config.test.yaml" +_SOURCE_DATASET = _TESTS_DIR.parent / "_docs" / "00_problem" / "input_data" / "dataset" + + +def _hardlink_tree(src_dir: Path, dst_dir: Path): + dst_dir.mkdir(parents=True, exist_ok=True) + for f in src_dir.iterdir(): + if f.is_file(): + target = dst_dir / f.name + if not target.exists(): + os.link(f, target) @pytest.fixture(scope="module") def e2e_result(): + # Arrange + src_images = _SOURCE_DATASET / "images" + src_labels = _SOURCE_DATASET / "labels" + if not src_images.is_dir() or not src_labels.is_dir(): + pytest.skip("source dataset not found") + old_config = c.config c.config = c.Config.from_yaml(str(_CONFIG_TEST), root=str(_TEST_ROOT)) + dst_images = Path(c.config.images_dir) + dst_labels = Path(c.config.labels_dir) + for d in (dst_images, dst_labels, c.config.datasets_dir, c.config.models_dir, c.config.corrupted_dir): + shutil.rmtree(str(d), ignore_errors=True) + + _hardlink_tree(src_images, dst_images) + _hardlink_tree(src_labels, dst_labels) + + linked_count = len(list(dst_images.glob("*.jpg"))) + Path(c.config.models_dir).mkdir(parents=True, exist_ok=True) + # Act train_mod.train_dataset() exports_mod.export_onnx(c.config.current_pt_model) @@ -31,8 +59,11 @@ def e2e_result(): yield { "today_dataset": today_ds, + "linked_count": linked_count, } + shutil.rmtree(str(dst_images), ignore_errors=True) + shutil.rmtree(str(dst_labels), ignore_errors=True) shutil.rmtree(c.config.datasets_dir, ignore_errors=True) shutil.rmtree(c.config.models_dir, ignore_errors=True) shutil.rmtree(c.config.corrupted_dir, ignore_errors=True) @@ -42,6 +73,7 @@ def e2e_result(): @pytest.mark.e2e class TestTrainingPipeline: def test_dataset_formed(self, e2e_result): + # Assert base = Path(e2e_result["today_dataset"]) for split in ("train", "valid", "test"): assert (base / split / "images").is_dir() @@ -50,7 +82,7 @@ class TestTrainingPipeline: len(list((base / s / "images").glob("*.jpg"))) for s in ("train", "valid", "test") ) - assert total == 20 + assert 0 < total <= e2e_result["linked_count"] def test_data_yaml_created(self, e2e_result): yaml_path = Path(e2e_result["today_dataset"]) / "data.yaml"