azaion/ai-training
1
0
Fork 0
mirror of https://github.com/azaion/ai-training.git synced 2026-04-22 11:16:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor constants management to use Pydantic BaseModel for configuration

Browse Source 
- Replaced module-level path variables in constants.py with a structured Pydantic Config class.
- Updated all relevant modules (train.py, augmentation.py, exports.py, dataset-visualiser.py, manual_run.py) to access paths through the new config structure.
- Fixed bugs related to image processing and model saving.
- Enhanced test infrastructure to accommodate the new configuration approach.

This refactor improves code maintainability and clarity by centralizing configuration management.
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-03-27 18:18:30 +02:00
parent b68c07b540
commit 142c6c4de8
106 changed files with 5706 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_dataset_formation.py
+24 -34
View File
@@ -55,8 +55,8 @@ def _prepare_form_dataset(
constants_patch(tmp_path)
import train
proc_img = Path(c_mod.processed_images_dir)
proc_lbl = Path(c_mod.processed_labels_dir)
proc_img = Path(c_mod.config.processed_images_dir)
proc_lbl = Path(c_mod.config.processed_labels_dir)
proc_img.mkdir(parents=True, exist_ok=True)
proc_lbl.mkdir(parents=True, exist_ok=True)
@@ -69,14 +69,8 @@ def _prepare_form_dataset(
if stem in corrupt_stems:
dst.write_text("0 1.5 0.5 0.1 0.1\n", encoding="utf-8")
today_ds = osp.join(c_mod.datasets_dir, train.today_folder)
monkeypatch.setattr(train, "today_dataset", today_ds)
monkeypatch.setattr(train, "processed_images_dir", c_mod.processed_images_dir)
monkeypatch.setattr(train, "processed_labels_dir", c_mod.processed_labels_dir)
monkeypatch.setattr(train, "corrupted_images_dir", c_mod.corrupted_images_dir)
monkeypatch.setattr(train, "corrupted_labels_dir", c_mod.corrupted_labels_dir)
monkeypatch.setattr(train, "datasets_dir", c_mod.datasets_dir)
return train
today_ds = osp.join(c_mod.config.datasets_dir, train.today_folder)
return train, today_ds
def _count_jpg(p):
@@ -90,7 +84,7 @@ def test_bt_dsf_01_split_ratio_70_20_10(
fixture_images_dir,
fixture_labels_dir,
):
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -100,10 +94,9 @@ def test_bt_dsf_01_split_ratio_70_20_10(
set(),
)
train.form_dataset()
base = train.today_dataset
assert _count_jpg(Path(base, "train", "images")) == 70
assert _count_jpg(Path(base, "valid", "images")) == 20
assert _count_jpg(Path(base, "test", "images")) == 10
assert _count_jpg(Path(today_ds, "train", "images")) == 70
assert _count_jpg(Path(today_ds, "valid", "images")) == 20
assert _count_jpg(Path(today_ds, "test", "images")) == 10
def test_bt_dsf_02_six_subdirectories(
@@ -113,7 +106,7 @@ def test_bt_dsf_02_six_subdirectories(
fixture_images_dir,
fixture_labels_dir,
):
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -123,7 +116,7 @@ def test_bt_dsf_02_six_subdirectories(
set(),
)
train.form_dataset()
base = Path(train.today_dataset)
base = Path(today_ds)
assert (base / "train" / "images").is_dir()
assert (base / "train" / "labels").is_dir()
assert (base / "valid" / "images").is_dir()
@@ -139,7 +132,7 @@ def test_bt_dsf_03_total_files_one_hundred(
fixture_images_dir,
fixture_labels_dir,
):
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -149,11 +142,10 @@ def test_bt_dsf_03_total_files_one_hundred(
set(),
)
train.form_dataset()
base = train.today_dataset
n = (
_count_jpg(Path(base, "train", "images"))
+ _count_jpg(Path(base, "valid", "images"))
+ _count_jpg(Path(base, "test", "images"))
_count_jpg(Path(today_ds, "train", "images"))
+ _count_jpg(Path(today_ds, "valid", "images"))
+ _count_jpg(Path(today_ds, "test", "images"))
)
assert n == 100
@@ -167,7 +159,7 @@ def test_bt_dsf_04_corrupted_labels_quarantined(
):
stems = [p.stem for p in sorted(fixture_images_dir.glob("*.jpg"))[:100]]
corrupt = set(stems[:5])
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -177,15 +169,14 @@ def test_bt_dsf_04_corrupted_labels_quarantined(
corrupt,
)
train.form_dataset()
base = train.today_dataset
split_total = (
_count_jpg(Path(base, "train", "images"))
+ _count_jpg(Path(base, "valid", "images"))
+ _count_jpg(Path(base, "test", "images"))
_count_jpg(Path(today_ds, "train", "images"))
+ _count_jpg(Path(today_ds, "valid", "images"))
+ _count_jpg(Path(today_ds, "test", "images"))
)
assert split_total == 95
assert _count_jpg(c_mod.corrupted_images_dir) == 5
assert len(list(Path(c_mod.corrupted_labels_dir).glob("*.txt"))) == 5
assert _count_jpg(c_mod.config.corrupted_images_dir) == 5
assert len(list(Path(c_mod.config.corrupted_labels_dir).glob("*.txt"))) == 5
@pytest.mark.resilience
@@ -196,7 +187,7 @@ def test_rt_dsf_01_empty_processed_no_crash(
fixture_images_dir,
fixture_labels_dir,
):
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -206,8 +197,7 @@ def test_rt_dsf_01_empty_processed_no_crash(
set(),
)
train.form_dataset()
base = Path(train.today_dataset)
assert base.is_dir()
assert Path(today_ds).is_dir()
@pytest.mark.resource_limit
@@ -225,7 +215,7 @@ def test_rl_dsf_02_no_filename_duplication_across_splits(
fixture_images_dir,
fixture_labels_dir,
):
train = _prepare_form_dataset(
train, today_ds = _prepare_form_dataset(
monkeypatch,
tmp_path,
constants_patch,
@@ -235,7 +225,7 @@ def test_rl_dsf_02_no_filename_duplication_across_splits(
set(),
)
train.form_dataset()
base = Path(train.today_dataset)
base = Path(today_ds)
names = []
for split in ("train", "valid", "test"):
for f in (base / split / "images").glob("*.jpg"):