mirror of
https://github.com/azaion/ai-training.git
synced 2026-04-23 06:46:36 +00:00
Refactor constants management to use Pydantic BaseModel for configuration
- Replaced module-level path variables in constants.py with a structured Pydantic Config class. - Updated all relevant modules (train.py, augmentation.py, exports.py, dataset-visualiser.py, manual_run.py) to access paths through the new config structure. - Fixed bugs related to image processing and model saving. - Enhanced test infrastructure to accommodate the new configuration approach. This refactor improves code maintainability and clarity by centralizing configuration management.
This commit is contained in:
@@ -20,15 +20,7 @@ if "matplotlib" not in sys.modules:
|
||||
|
||||
|
||||
def _patch_augmentation_paths(monkeypatch, base: Path):
|
||||
import augmentation as aug
|
||||
import constants as c
|
||||
|
||||
apply_constants_patch(monkeypatch, base)
|
||||
monkeypatch.setattr(aug, "data_images_dir", c.data_images_dir)
|
||||
monkeypatch.setattr(aug, "data_labels_dir", c.data_labels_dir)
|
||||
monkeypatch.setattr(aug, "processed_images_dir", c.processed_images_dir)
|
||||
monkeypatch.setattr(aug, "processed_labels_dir", c.processed_labels_dir)
|
||||
monkeypatch.setattr(aug, "processed_dir", c.processed_dir)
|
||||
|
||||
|
||||
def _augment_annotation_with_total(monkeypatch):
|
||||
@@ -58,8 +50,8 @@ def test_pt_aug_01_throughput_ten_images_sixty_seconds(
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.data_images_dir)
|
||||
lbl_dir = Path(c.data_labels_dir)
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(10)
|
||||
@@ -83,9 +75,9 @@ def test_pt_aug_02_parallel_at_least_one_point_five_x_faster(
|
||||
import constants as c
|
||||
from augmentation import Augmentator
|
||||
|
||||
img_dir = Path(c.data_images_dir)
|
||||
lbl_dir = Path(c.data_labels_dir)
|
||||
proc_dir = Path(c.processed_dir)
|
||||
img_dir = Path(c.config.data_images_dir)
|
||||
lbl_dir = Path(c.config.data_labels_dir)
|
||||
proc_dir = Path(c.config.processed_dir)
|
||||
img_dir.mkdir(parents=True, exist_ok=True)
|
||||
lbl_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_img, src_lbl = sample_images_labels(10)
|
||||
@@ -93,8 +85,8 @@ def test_pt_aug_02_parallel_at_least_one_point_five_x_faster(
|
||||
shutil.copy2(p, img_dir / p.name)
|
||||
for p in src_lbl.glob("*.txt"):
|
||||
shutil.copy2(p, lbl_dir / p.name)
|
||||
Path(c.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
names = sorted(p.name for p in img_dir.glob("*.jpg"))
|
||||
|
||||
class _E:
|
||||
@@ -113,8 +105,8 @@ def test_pt_aug_02_parallel_at_least_one_point_five_x_faster(
|
||||
seq_elapsed = time.perf_counter() - t0
|
||||
|
||||
shutil.rmtree(proc_dir)
|
||||
Path(c.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_images_dir).mkdir(parents=True, exist_ok=True)
|
||||
Path(c.config.processed_labels_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
aug_par = Augmentator()
|
||||
aug_par.total_images_to_process = len(entries)
|
||||
|
||||
@@ -56,8 +56,8 @@ def _prepare_form_dataset(
|
||||
constants_patch(tmp_path)
|
||||
import train
|
||||
|
||||
proc_img = Path(c_mod.processed_images_dir)
|
||||
proc_lbl = Path(c_mod.processed_labels_dir)
|
||||
proc_img = Path(c_mod.config.processed_images_dir)
|
||||
proc_lbl = Path(c_mod.config.processed_labels_dir)
|
||||
proc_img.mkdir(parents=True, exist_ok=True)
|
||||
proc_lbl.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -70,14 +70,8 @@ def _prepare_form_dataset(
|
||||
if stem in corrupt_stems:
|
||||
dst.write_text("0 1.5 0.5 0.1 0.1\n", encoding="utf-8")
|
||||
|
||||
today_ds = osp.join(c_mod.datasets_dir, train.today_folder)
|
||||
monkeypatch.setattr(train, "today_dataset", today_ds)
|
||||
monkeypatch.setattr(train, "processed_images_dir", c_mod.processed_images_dir)
|
||||
monkeypatch.setattr(train, "processed_labels_dir", c_mod.processed_labels_dir)
|
||||
monkeypatch.setattr(train, "corrupted_images_dir", c_mod.corrupted_images_dir)
|
||||
monkeypatch.setattr(train, "corrupted_labels_dir", c_mod.corrupted_labels_dir)
|
||||
monkeypatch.setattr(train, "datasets_dir", c_mod.datasets_dir)
|
||||
return train
|
||||
today_ds = osp.join(c_mod.config.datasets_dir, train.today_folder)
|
||||
return train, today_ds
|
||||
|
||||
|
||||
@pytest.mark.performance
|
||||
@@ -88,7 +82,7 @@ def test_pt_dsf_01_dataset_formation_under_thirty_seconds(
|
||||
fixture_images_dir,
|
||||
fixture_labels_dir,
|
||||
):
|
||||
train = _prepare_form_dataset(
|
||||
train, today_ds = _prepare_form_dataset(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
constants_patch,
|
||||
|
||||
Reference in New Issue
Block a user