mirror of
https://github.com/azaion/ai-training.git
synced 2026-06-21 10:31:11 +00:00
- Removed unused imports and the `form_data_sample` function from `exports.py` to streamline the code. - Added whitespace in `constants.py` for improved readability. These changes enhance code clarity and maintainability.
This commit is contained in:
+2
-1
@@ -54,7 +54,8 @@ class Config(BaseModel):
|
|||||||
@property
|
@property
|
||||||
def data_dir(self) -> str:
|
def data_dir(self) -> str:
|
||||||
return path.join(self.dirs.root, self.dirs.data)
|
return path.join(self.dirs.root, self.dirs.data)
|
||||||
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def images_dir(self) -> str:
|
def images_dir(self) -> str:
|
||||||
return path.join(self.data_dir, IMAGES_DIR)
|
return path.join(self.data_dir, IMAGES_DIR)
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import argparse
|
||||||
|
import random
|
||||||
|
import shutil
|
||||||
|
from os import path, scandir, makedirs
|
||||||
|
|
||||||
|
import constants
|
||||||
|
|
||||||
|
|
||||||
|
def form_data_sample(destination_path: str = None, size: int = 500, write_txt_log: bool = False) -> None:
|
||||||
|
if destination_path is None:
|
||||||
|
destination_path = constants.config.sample_dir
|
||||||
|
|
||||||
|
images = []
|
||||||
|
with scandir(constants.config.images_dir) as imd:
|
||||||
|
for image_file in imd:
|
||||||
|
if not image_file.is_file():
|
||||||
|
continue
|
||||||
|
images.append(image_file)
|
||||||
|
print('shuffling images')
|
||||||
|
random.shuffle(images)
|
||||||
|
images = images[:size]
|
||||||
|
|
||||||
|
shutil.rmtree(destination_path, ignore_errors=True)
|
||||||
|
makedirs(destination_path, exist_ok=True)
|
||||||
|
|
||||||
|
lines = []
|
||||||
|
for image in images:
|
||||||
|
shutil.copy(image.path, path.join(destination_path, image.name))
|
||||||
|
lines.append(f'./{image.name}')
|
||||||
|
if write_txt_log:
|
||||||
|
with open(path.join(destination_path, 'azaion_subset.txt'), 'w', encoding='utf-8') as f:
|
||||||
|
f.writelines([f'{line}\n' for line in lines])
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Form a random image sample (e.g. for int8 calibration).',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--dest', '-d',
|
||||||
|
default=None,
|
||||||
|
help='Destination directory. Defaults to constants.config.sample_dir. WARNING: wiped before writing.',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--size', '-n',
|
||||||
|
type=int,
|
||||||
|
default=500,
|
||||||
|
help='Number of images to sample (default: 500).',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--write-txt-log',
|
||||||
|
action='store_true',
|
||||||
|
help='Also write azaion_subset.txt listing the sampled file names.',
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
args = _parse_args()
|
||||||
|
form_data_sample(
|
||||||
|
destination_path=args.dest,
|
||||||
|
size=args.size,
|
||||||
|
write_txt_log=args.write_txt_log,
|
||||||
|
)
|
||||||
+1
-25
@@ -1,8 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
from os import path, scandir, makedirs
|
from os import path
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import random
|
|
||||||
|
|
||||||
import netron
|
import netron
|
||||||
import yaml
|
import yaml
|
||||||
@@ -60,29 +59,6 @@ def export_tensorrt(model_path):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def form_data_sample(destination_path, size=500, write_txt_log=False):
|
|
||||||
images = []
|
|
||||||
with scandir(constants.config.images_dir) as imd:
|
|
||||||
for image_file in imd:
|
|
||||||
if not image_file.is_file():
|
|
||||||
continue
|
|
||||||
images.append(image_file)
|
|
||||||
print('shuffling images')
|
|
||||||
random.shuffle(images)
|
|
||||||
images = images[:size]
|
|
||||||
|
|
||||||
shutil.rmtree(destination_path, ignore_errors=True)
|
|
||||||
makedirs(destination_path, exist_ok=True)
|
|
||||||
|
|
||||||
lines = []
|
|
||||||
for image in images:
|
|
||||||
shutil.copy(image.path, path.join(destination_path, image.name))
|
|
||||||
lines.append(f'./{image.name}')
|
|
||||||
if write_txt_log:
|
|
||||||
with open(path.join(destination_path, 'azaion_subset.txt'), 'w', encoding='utf-8') as f:
|
|
||||||
f.writelines([f'{line}\n' for line in lines])
|
|
||||||
|
|
||||||
|
|
||||||
def show_model(model: str = None):
|
def show_model(model: str = None):
|
||||||
netron.start(model)
|
netron.start(model)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user