From eb5fe5348c454e59e4f52b7d2aebc0674e19ebf2 Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 12:10:07 +0900 Subject: [PATCH 01/96] =?UTF-8?q?=EA=B0=81=20=EC=93=B0=EB=A0=88=EB=93=9C?= =?UTF-8?q?=20=EC=8B=9C=EA=B0=84=20=EC=B8=A1=EC=A0=95=20=EB=94=94=EB=B2=84?= =?UTF-8?q?=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index 0024789..b800afa 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -98,9 +98,10 @@ def ModelRequest(model, data): res = requests.post(url, data, headers) response = json.loads(res.text)['predictions'] - end_time = time.time() - inference_start + inference_end = time.time() + end_time = inference_end - inference_start - key = model + " " + str(inference_start) + key = model + " " + str(inference_start) + " " + str(inference_end) time_tracking[key] = end_time return response @@ -130,7 +131,7 @@ def ModelRequest(model, data): # ModelRequest(model, datas[model]) while True: - if time.time() - event_start >=1: + if time.time() - event_start >= 1: break for thread in threads: From d02715b3b835ae9ea2b598242633a0009bed06f5 Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 12:45:24 +0900 Subject: [PATCH 02/96] =?UTF-8?q?=EA=B0=81=20=EC=93=B0=EB=A0=88=EB=93=9C?= =?UTF-8?q?=20=EC=8B=9C=EA=B0=84=20=EC=B8=A1=EC=A0=95=20=EB=94=94=EB=B2=84?= =?UTF-8?q?=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index b800afa..3203b82 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -110,7 +110,7 @@ def ModelRequest(model, data): get_weighted_smooth = roundrobin.smooth(models) model_sequence = [get_weighted_smooth() for _ in range(MAX)] RoundPerEvent = 10 -TotalEvent = 10 +TotalEvent = 1 poisson_distribution = random.poisson(RoundPerEvent, TotalEvent) if __name__ == "__main__": @@ -118,7 +118,6 @@ def ModelRequest(model, data): threads = [] for events in poisson_distribution: - print('request', events) event_start = time.time() for model_idx in range(events): @@ -137,7 +136,6 @@ def ModelRequest(model, data): for thread in threads: thread.join() request_end = time.time() - request_start - print("Return value:", poisson_distribution) print("Length of return value:", len(poisson_distribution)) print("total request time", request_end) From 9292100733aeaa0d31ac318037af95bf5b4d4cdf Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 12:46:37 +0900 Subject: [PATCH 03/96] =?UTF-8?q?=EC=9A=94=EC=B2=AD=20=EC=A4=84=EC=97=AC?= =?UTF-8?q?=20=EB=94=94=EB=B2=84=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index 3203b82..c20267a 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -109,7 +109,7 @@ def ModelRequest(model, data): get_weighted_smooth = roundrobin.smooth(models) model_sequence = [get_weighted_smooth() for _ in range(MAX)] -RoundPerEvent = 10 +RoundPerEvent = 5 TotalEvent = 1 poisson_distribution = random.poisson(RoundPerEvent, TotalEvent) @@ -119,7 +119,7 @@ def ModelRequest(model, data): for events in poisson_distribution: event_start = time.time() - + print('request', events) for model_idx in range(events): model = model_sequence[model_idx % len(model_sequence)] th = Thread(target=ModelRequest, args=(model, datas[model])) From bcbe536a751aaa64a9bc3af7e639e96f64312df0 Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 12:48:41 +0900 Subject: [PATCH 04/96] =?UTF-8?q?sleep=20=EB=8A=98=EB=A0=A4=20=EC=8B=A4?= =?UTF-8?q?=ED=97=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index c20267a..8351769 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -110,9 +110,10 @@ def ModelRequest(model, data): get_weighted_smooth = roundrobin.smooth(models) model_sequence = [get_weighted_smooth() for _ in range(MAX)] RoundPerEvent = 5 -TotalEvent = 1 +TotalEvent = 3 poisson_distribution = random.poisson(RoundPerEvent, TotalEvent) +sleep_val = 10 if __name__ == "__main__": request_start = time.time() threads = [] @@ -130,7 +131,7 @@ def ModelRequest(model, data): # ModelRequest(model, datas[model]) while True: - if time.time() - event_start >= 1: + if time.time() - event_start >= sleep_val: break for thread in threads: From 5de2cb49db935ac1ff393650924cc76d10a9c5ec Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 13:14:30 +0900 Subject: [PATCH 05/96] =?UTF-8?q?=EB=94=94=EB=B2=84=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index 8351769..52780de 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -109,11 +109,11 @@ def ModelRequest(model, data): get_weighted_smooth = roundrobin.smooth(models) model_sequence = [get_weighted_smooth() for _ in range(MAX)] -RoundPerEvent = 5 -TotalEvent = 3 +RoundPerEvent = 10 +TotalEvent = 5 poisson_distribution = random.poisson(RoundPerEvent, TotalEvent) -sleep_val = 10 +sleep_val = 1 if __name__ == "__main__": request_start = time.time() threads = [] From c0431408273d16a78bbb66b527f195c2b174f744 Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 13:27:57 +0900 Subject: [PATCH 06/96] =?UTF-8?q?=EB=94=94=EB=B2=84=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index 52780de..fffed05 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -110,7 +110,7 @@ def ModelRequest(model, data): get_weighted_smooth = roundrobin.smooth(models) model_sequence = [get_weighted_smooth() for _ in range(MAX)] RoundPerEvent = 10 -TotalEvent = 5 +TotalEvent = 1 poisson_distribution = random.poisson(RoundPerEvent, TotalEvent) sleep_val = 1 From b8bfec017f30ac791a20d85ffb89f35ea4731501 Mon Sep 17 00:00:00 2001 From: manchann Date: Tue, 14 Feb 2023 13:38:21 +0900 Subject: [PATCH 07/96] =?UTF-8?q?=EB=94=94=EB=B2=84=EA=B9=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/roundrobin_poisson_inference_request.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CNN/roundrobin_poisson_inference_request.py b/CNN/roundrobin_poisson_inference_request.py index fffed05..1f2abb4 100644 --- a/CNN/roundrobin_poisson_inference_request.py +++ b/CNN/roundrobin_poisson_inference_request.py @@ -102,6 +102,9 @@ def ModelRequest(model, data): end_time = inference_end - inference_start key = model + " " + str(inference_start) + " " + str(inference_end) + + print(key, end_time) + time_tracking[key] = end_time return response From d2d2b7d5f1ee3ad5207a2eaa0e4f85c79cba2c66 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 16 Feb 2023 15:44:02 +0900 Subject: [PATCH 08/96] =?UTF-8?q?edge=20=EC=9E=A5=EB=B9=84=EC=97=90=20?= =?UTF-8?q?=EC=B6=94=EB=A1=A0=EC=9D=84=20=EC=9A=94=EC=B2=AD=ED=95=98?= =?UTF-8?q?=EB=8A=94=20=EB=AA=A8=EB=93=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 105 ++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 CNN/edge_inference_request.py diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py new file mode 100644 index 0000000..682a987 --- /dev/null +++ b/CNN/edge_inference_request.py @@ -0,0 +1,105 @@ +from flask import Flask +import tensorflow as tf +import numpy as np +import psutil +import shutil + +gpus = tf.config.experimental.list_physical_devices('GPU') +if gpus: + tf.config.experimental.set_virtual_device_configuration(gpus[0], + [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.18*1024)]) + + print("1:", psutil.virtual_memory()) + +from tensorflow.keras.applications import ( + mobilenet, + mobilenet_v2, + inception_v3 + ) + +models = { + 'mobilenet':mobilenet, + 'mobilenet_v2':mobilenet_v2, + 'inception_v3':inception_v3 + } + +models_detail = { + 'mobilenet':mobilenet.MobileNet(weights='imagenet'), + 'mobilenet_v2':mobilenet_v2.MobileNetV2(weights='imagenet'), + 'inception_v3':inception_v3.InceptionV3(weights='imagenet') + } + +def mobilenet_load_image(image_path): + return tf.keras.preprocessing.image.load_img( + image_path, + target_size=[224, 224]) + + +def inception_load_image(image_path): + return tf.keras.preprocessing.image.load_img( + image_path, + target_size=[299, 299]) + + +def image_to_array(image): + return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) + +def image_preprocess(image_array, model_name): + return models[model_name].preprocess_input( + image_array[tf.newaxis, ...]) + +# def mobilenetv1_image_preprocess(image_array): +# return mobilenet.preprocess_input( +# image_array[tf.newaxis, ...]) +# +# +# def mobilenetv2_image_preprocess(image_array): +# return mobilenet_v2.preprocess_input( +# image_array[tf.newaxis, ...]) +# +# +# def inceptionv3_image_preprocess(image_array): +# return inception_v3.preprocess_input( +# image_array[tf.newaxis, ...]) + +mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' +mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' +inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' + +mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) +mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) +inceptionv3_test_image = inception_load_image(inceptionv3_image_path) + +mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) +mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) +inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) + +# mobilenetv1_test_image_preprocessed = mobilenetv1_image_preprocess(mobilenetv1_test_image_array) +# mobilenetv2_test_image_preprocessed = mobilenetv2_image_preprocess(mobilenetv2_test_image_array) +# inceptionv3_test_image_preprocessed = inceptionv3_image_preprocess(inceptionv3_test_image_array) + +mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') +mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') +inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') + + +def save_model(model, saved_model_dir): + model = models_detail[model] + shutil.rmtree(saved_model_dir, ignore_errors=True) + model.save(saved_model_dir, include_optimizer=False, save_format='tf') + +loaded_models = {} + +for model in models.keys(): + path = f'{model}_saved_model' + save_model(model, path) + loaded_models[model] = tf.keras.models.load_model(path) + +app = Flask(__name__) + +@app.route('/mobilenetv1') +def mobilenetv1(): + result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + return result + +app.run(host='localhost', port=5001) \ No newline at end of file From c5cfc1cdfee9d1b72756bf1dc70c5f2d5d123b59 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 16 Feb 2023 17:24:42 +0900 Subject: [PATCH 09/96] =?UTF-8?q?gpu=20=EB=A9=94=EB=AA=A8=EB=A6=AC=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=EB=9F=89=20=EC=A6=9D=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 682a987..030d1e0 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -7,9 +7,9 @@ gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: tf.config.experimental.set_virtual_device_configuration(gpus[0], - [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.18*1024)]) + [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.3*1024)]) - print("1:", psutil.virtual_memory()) + #print("1:", psutil.virtual_memory()) from tensorflow.keras.applications import ( mobilenet, @@ -18,15 +18,15 @@ ) models = { - 'mobilenet':mobilenet, - 'mobilenet_v2':mobilenet_v2, - 'inception_v3':inception_v3 + 'mobilenet': mobilenet, + 'mobilenet_v2': mobilenet_v2, + 'inception_v3': inception_v3 } models_detail = { - 'mobilenet':mobilenet.MobileNet(weights='imagenet'), - 'mobilenet_v2':mobilenet_v2.MobileNetV2(weights='imagenet'), - 'inception_v3':inception_v3.InceptionV3(weights='imagenet') + 'mobilenet': mobilenet.MobileNet(weights='imagenet'), + 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), + 'inception_v3': inception_v3.InceptionV3(weights='imagenet') } def mobilenet_load_image(image_path): From 708587f1a8bbbaa559e97728d46531ddd61a77d6 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:42:32 +0900 Subject: [PATCH 10/96] Rename raw_image_classification_inference.py to cnn_raw_image_classification_inference.py file name change --- ...ion_inference.py => cnn_raw_image_classification_inference.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CNN/{raw_image_classification_inference.py => cnn_raw_image_classification_inference.py} (100%) diff --git a/CNN/raw_image_classification_inference.py b/CNN/cnn_raw_image_classification_inference.py similarity index 100% rename from CNN/raw_image_classification_inference.py rename to CNN/cnn_raw_image_classification_inference.py From 7c7db74ef818f716273830d90a2a7e21c0d6f718 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:44:50 +0900 Subject: [PATCH 11/96] Create coral-edgetpu-tflite-aw_image_classification_inference.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit edgetpu-tflite 모델 raw image 추론 코드 --- ...flite-aw_image_classification_inference.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 CNN/coral-edgetpu-tflite-aw_image_classification_inference.py diff --git a/CNN/coral-edgetpu-tflite-aw_image_classification_inference.py b/CNN/coral-edgetpu-tflite-aw_image_classification_inference.py new file mode 100644 index 0000000..584f103 --- /dev/null +++ b/CNN/coral-edgetpu-tflite-aw_image_classification_inference.py @@ -0,0 +1,91 @@ +import argparse +import time +import os +import numpy as np + +from PIL import Image + +from model import classify +import tflite_runtime.interpreter as tflite +import platform + +EDGETPU_SHARED_LIB = { + 'Linux': 'libedgetpu.so.1', + 'Darwin': 'libedgetpu.1.dylib', + 'Windows': 'edgetpu.dll' +}[platform.system()] + + +def load_labels(path, encoding='utf-8'): + with open(path, 'r', encoding=encoding) as f: + lines = f.readlines() + if not lines: + return {} + + if lines[0].split(' ', maxsplit=1)[0].isdigit(): + pairs = [line.split(' ', maxsplit=1) for line in lines] + return {int(index): label.strip() for index, label in pairs} + else: + return {index: line.strip() for index, line in enumerate(lines)} + + +def make_interpreter(model_file): + model_file, *device = model_file.split('@') + return tflite.Interpreter( + model_path=model_file, + experimental_delegates=[ + tflite.load_delegate(EDGETPU_SHARED_LIB, + {'device': device[0]} if device else {}) + ]) + + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '-m', '--model', required=True, help='File path of .tflite file.') + parser.add_argument( + '-l', '--labels', help='File path of labels file.') + parser.add_argument( + '-k', '--top_k', type=int, default=1, + help='Max number of classification results') + parser.add_argument( + '-t', '--threshold', type=float, default=0.0, + help='Classification score threshold') + args = parser.parse_args() + + labels = load_labels(args.labels) if args.labels else {} + + model_load_time = time.time() + interpreter = make_interpreter(args.model) + interpreter.allocate_tensors() + model_load_time = time.time() - model_load_time + + dataset_load_time=time.time() + image = Image.open('./dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG') + dataset_load_time = time.time() - dataset_load_time + + + size = classify.input_size(interpreter) + image = image.convert('RGB').resize(size, Image.ANTIALIAS) + classify.set_input(interpreter, image) + start = time.perf_counter() + interpreter.invoke() + iter_times = time.perf_counter() - start + classes = classify.get_output(interpreter, args.top_k, args.threshold) + for klass in classes: + accuracy = klass.score + + + print('***** TF-lite matric *****') + print('accuracy =', accuracy) + print('model_load_time =', model_load_time) + print('dataset_load_time =', dataset_load_time) + print('inference_time =', iter_times) + print('IPS =', 1 / (model_load_time + dataset_load_time + iter_times)) + print('IPS(inf) =', 1 / iter_times) + + +if __name__ == '__main__': + main() From 88846be48fbd9f5a008571d383be5e283f277f85 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:45:16 +0900 Subject: [PATCH 12/96] Rename coral-edgetpu-tflite-aw_image_classification_inference.py to coral-edgetpu-tflite-raw_image_classification_inference.py file name change --- ...=> coral-edgetpu-tflite-raw_image_classification_inference.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CNN/{coral-edgetpu-tflite-aw_image_classification_inference.py => coral-edgetpu-tflite-raw_image_classification_inference.py} (100%) diff --git a/CNN/coral-edgetpu-tflite-aw_image_classification_inference.py b/CNN/coral-edgetpu-tflite-raw_image_classification_inference.py similarity index 100% rename from CNN/coral-edgetpu-tflite-aw_image_classification_inference.py rename to CNN/coral-edgetpu-tflite-raw_image_classification_inference.py From a7f436be273c6a02c74978916f2a0d1f311eac57 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:46:39 +0900 Subject: [PATCH 13/96] Create object_detect_raw_image_classification_inference.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yolov5 모델 raw image 추론 코드 --- ...tect_raw_image_classification_inference.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 CNN/object_detect_raw_image_classification_inference.py diff --git a/CNN/object_detect_raw_image_classification_inference.py b/CNN/object_detect_raw_image_classification_inference.py new file mode 100644 index 0000000..e69a1e5 --- /dev/null +++ b/CNN/object_detect_raw_image_classification_inference.py @@ -0,0 +1,107 @@ +import argparse +import os +import platform +import sys +from pathlib import Path + +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.plots import Annotator, colors, save_one_box +from utils.torch_utils import select_device, smart_inference_mode + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + vid_stride=1, # video frame-rate stride +): + source = str(source) + + # Load model + # device = select_device(device) + model = DetectMultiBackend(weights, data=data) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + pred = model(im) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) + + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + p = Path(p) # to Path + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s.pt', help='model path or triton URL') + parser.add_argument('--source', type=str, default=ROOT / 'data/images', help='file/dir/URL/glob/screen/0(webcam)') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + + +def main(opt): + run(**vars(opt)) + + +if __name__ == "__main__": + opt = parse_opt() + main(opt) From 643aa7690bdfa52059f069491a6843e3a3491238 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 16 Feb 2023 17:59:23 +0900 Subject: [PATCH 14/96] =?UTF-8?q?=EB=A9=94=EB=AA=A8=EB=A6=AC=EC=82=AC?= =?UTF-8?q?=EC=9D=B4=EC=A6=88=20=EC=A6=9D=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 030d1e0..3d7e363 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -7,7 +7,7 @@ gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: tf.config.experimental.set_virtual_device_configuration(gpus[0], - [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.3*1024)]) + [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.6*1024)]) #print("1:", psutil.virtual_memory()) From b0d8484bed7ec7fe1e254407c7cf5cbf2d095c33 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 16 Feb 2023 18:57:52 +0900 Subject: [PATCH 15/96] =?UTF-8?q?mobilenet=EB=A7=8C=20=EB=A1=9C=EB=93=9C?= =?UTF-8?q?=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 3d7e363..3e8b87c 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -90,10 +90,14 @@ def save_model(model, saved_model_dir): loaded_models = {} -for model in models.keys(): - path = f'{model}_saved_model' - save_model(model, path) - loaded_models[model] = tf.keras.models.load_model(path) +# for model in models.keys(): +# path = f'{model}_saved_model' +# save_model(model, path) +# loaded_models[model] = tf.keras.models.load_model(path) + +path = 'mobilenet_saved_model' +save_model('mobilenet', path) +loaded_models['mobilenet'] = tf.keras.models.load_model(path) app = Flask(__name__) From 370bc31d10765848ba3c827af658929e3d2f2758 Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 10:31:50 +0900 Subject: [PATCH 16/96] =?UTF-8?q?model=20save=20=EB=90=98=EC=96=B4?= =?UTF-8?q?=EC=9E=88=EC=9C=BC=EB=A9=B4=20save=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 3e8b87c..99178cd 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -3,6 +3,7 @@ import numpy as np import psutil import shutil +import os gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: @@ -95,9 +96,11 @@ def save_model(model, saved_model_dir): # save_model(model, path) # loaded_models[model] = tf.keras.models.load_model(path) -path = 'mobilenet_saved_model' -save_model('mobilenet', path) -loaded_models['mobilenet'] = tf.keras.models.load_model(path) +model_path = 'mobilenet_saved_model' +if os.path.isdir(model_path) == False: + print('model save') + save_model('mobilenet', model_path) +loaded_models['mobilenet'] = tf.keras.models.load_model(model_path) app = Flask(__name__) From c1c0bb7b0d33494021cd90724408e92ede188473 Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 10:35:53 +0900 Subject: [PATCH 17/96] result debugging --- CNN/edge_inference_request.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 99178cd..6c4eb5c 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -107,6 +107,7 @@ def save_model(model, saved_model_dir): @app.route('/mobilenetv1') def mobilenetv1(): result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - return result + print(result) + return 'mobilenetv1 inference success' app.run(host='localhost', port=5001) \ No newline at end of file From eb8c6da740ef2d63f264c04d8f29555b22907947 Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 11:30:08 +0900 Subject: [PATCH 18/96] =?UTF-8?q?=EC=84=B8=EA=B0=9C=EC=9D=98=20=EB=AA=A8?= =?UTF-8?q?=EB=8D=B8=20=EC=84=9C=EB=B2=84=20=EB=B0=B0=ED=8F=AC=20=EC=A7=84?= =?UTF-8?q?=ED=96=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 61 +++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 6c4eb5c..22fe913 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -7,28 +7,30 @@ gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: - tf.config.experimental.set_virtual_device_configuration(gpus[0], - [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=0.6*1024)]) + tf.config.experimental.set_virtual_device_configuration(gpus[0], + [tf.config.experimental.VirtualDeviceConfiguration( + memory_limit=0.6 * 1024)]) - #print("1:", psutil.virtual_memory()) + # print("1:", psutil.virtual_memory()) from tensorflow.keras.applications import ( - mobilenet, - mobilenet_v2, - inception_v3 - ) + mobilenet, + mobilenet_v2, + inception_v3 +) models = { - 'mobilenet': mobilenet, - 'mobilenet_v2': mobilenet_v2, - 'inception_v3': inception_v3 - } + 'mobilenet': mobilenet, + 'mobilenet_v2': mobilenet_v2, + 'inception_v3': inception_v3 +} models_detail = { - 'mobilenet': mobilenet.MobileNet(weights='imagenet'), - 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), - 'inception_v3': inception_v3.InceptionV3(weights='imagenet') - } + 'mobilenet': mobilenet.MobileNet(weights='imagenet'), + 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), + 'inception_v3': inception_v3.InceptionV3(weights='imagenet') +} + def mobilenet_load_image(image_path): return tf.keras.preprocessing.image.load_img( @@ -45,10 +47,12 @@ def inception_load_image(image_path): def image_to_array(image): return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) + def image_preprocess(image_array, model_name): return models[model_name].preprocess_input( image_array[tf.newaxis, ...]) + # def mobilenetv1_image_preprocess(image_array): # return mobilenet.preprocess_input( # image_array[tf.newaxis, ...]) @@ -89,6 +93,7 @@ def save_model(model, saved_model_dir): shutil.rmtree(saved_model_dir, ignore_errors=True) model.save(saved_model_dir, include_optimizer=False, save_format='tf') + loaded_models = {} # for model in models.keys(): @@ -97,17 +102,33 @@ def save_model(model, saved_model_dir): # loaded_models[model] = tf.keras.models.load_model(path) model_path = 'mobilenet_saved_model' -if os.path.isdir(model_path) == False: - print('model save') - save_model('mobilenet', model_path) -loaded_models['mobilenet'] = tf.keras.models.load_model(model_path) +model_names = models_detail.keys() +for model_name in model_names: + if os.path.isdir(model_path) == False: + print('model save') + save_model(model_name, model_name + '_saved_model') + loaded_models[model_name] = tf.keras.models.load_model(model_path) app = Flask(__name__) + @app.route('/mobilenetv1') def mobilenetv1(): result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) print(result) return 'mobilenetv1 inference success' -app.run(host='localhost', port=5001) \ No newline at end of file +@app.route('/mobilenetv2') +def mobilenetv1(): + result = loaded_models['mobilenet_v2'].predict(mobilenetv1_test_image_preprocessed) + print(result) + return 'mobilenetv2 inference success' + +@app.route('/inception_v3') +def mobilenetv1(): + result = loaded_models['inception_v3'].predict(mobilenetv1_test_image_preprocessed) + print(result) + return 'inceptionv3 inference success' + + +app.run(host='localhost', port=5001) From 21af7d4c41e67a52f465b8aec02ba34267cc911b Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 11:33:22 +0900 Subject: [PATCH 19/96] =?UTF-8?q?function=20name=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 22fe913..1da2dc6 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -119,13 +119,13 @@ def mobilenetv1(): return 'mobilenetv1 inference success' @app.route('/mobilenetv2') -def mobilenetv1(): +def mobilenetv2(): result = loaded_models['mobilenet_v2'].predict(mobilenetv1_test_image_preprocessed) print(result) return 'mobilenetv2 inference success' @app.route('/inception_v3') -def mobilenetv1(): +def mobilenetv3(): result = loaded_models['inception_v3'].predict(mobilenetv1_test_image_preprocessed) print(result) return 'inceptionv3 inference success' From d15e9eb0ac22a8d70dfd0c44c70d981afa8e9b7a Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 11:37:27 +0900 Subject: [PATCH 20/96] =?UTF-8?q?function=20name=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 1da2dc6..76708a2 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -124,8 +124,8 @@ def mobilenetv2(): print(result) return 'mobilenetv2 inference success' -@app.route('/inception_v3') -def mobilenetv3(): +@app.route('/inceptionv3') +def inceptionv3(): result = loaded_models['inception_v3'].predict(mobilenetv1_test_image_preprocessed) print(result) return 'inceptionv3 inference success' From 9a1adc1100e8b8896fab90ef8958a17e2120cd85 Mon Sep 17 00:00:00 2001 From: manchann Date: Fri, 17 Feb 2023 11:38:08 +0900 Subject: [PATCH 21/96] =?UTF-8?q?=EC=A3=BC=EC=84=9D=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 76708a2..15a8c96 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -7,9 +7,10 @@ gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: - tf.config.experimental.set_virtual_device_configuration(gpus[0], - [tf.config.experimental.VirtualDeviceConfiguration( - memory_limit=0.6 * 1024)]) + tf.config.experimental.set_virtual_device_configuration( + gpus[0], + [tf.config.experimental.VirtualDeviceConfiguration( + memory_limit=0.6 * 1024)]) # print("1:", psutil.virtual_memory()) @@ -53,20 +54,6 @@ def image_preprocess(image_array, model_name): image_array[tf.newaxis, ...]) -# def mobilenetv1_image_preprocess(image_array): -# return mobilenet.preprocess_input( -# image_array[tf.newaxis, ...]) -# -# -# def mobilenetv2_image_preprocess(image_array): -# return mobilenet_v2.preprocess_input( -# image_array[tf.newaxis, ...]) -# -# -# def inceptionv3_image_preprocess(image_array): -# return inception_v3.preprocess_input( -# image_array[tf.newaxis, ...]) - mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' @@ -79,10 +66,6 @@ def image_preprocess(image_array, model_name): mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) -# mobilenetv1_test_image_preprocessed = mobilenetv1_image_preprocess(mobilenetv1_test_image_array) -# mobilenetv2_test_image_preprocessed = mobilenetv2_image_preprocess(mobilenetv2_test_image_array) -# inceptionv3_test_image_preprocessed = inceptionv3_image_preprocess(inceptionv3_test_image_array) - mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') @@ -96,11 +79,6 @@ def save_model(model, saved_model_dir): loaded_models = {} -# for model in models.keys(): -# path = f'{model}_saved_model' -# save_model(model, path) -# loaded_models[model] = tf.keras.models.load_model(path) - model_path = 'mobilenet_saved_model' model_names = models_detail.keys() for model_name in model_names: @@ -118,12 +96,14 @@ def mobilenetv1(): print(result) return 'mobilenetv1 inference success' + @app.route('/mobilenetv2') def mobilenetv2(): result = loaded_models['mobilenet_v2'].predict(mobilenetv1_test_image_preprocessed) print(result) return 'mobilenetv2 inference success' + @app.route('/inceptionv3') def inceptionv3(): result = loaded_models['inception_v3'].predict(mobilenetv1_test_image_preprocessed) From c4347317021551079089507d7947da719ecc4b35 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 17 Feb 2023 12:01:38 +0900 Subject: [PATCH 22/96] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 15a8c96..367c2fe 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -1,7 +1,6 @@ from flask import Flask import tensorflow as tf import numpy as np -import psutil import shutil import os @@ -12,8 +11,6 @@ [tf.config.experimental.VirtualDeviceConfiguration( memory_limit=0.6 * 1024)]) - # print("1:", psutil.virtual_memory()) - from tensorflow.keras.applications import ( mobilenet, mobilenet_v2, From b93da0b99bbe1297cb21dd85f3168efba25bc840 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:44:11 +0900 Subject: [PATCH 23/96] Update dataset_download.sh dataset path change --- CNN/dataset/dataset_download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/dataset/dataset_download.sh b/CNN/dataset/dataset_download.sh index 2105943..c27f551 100644 --- a/CNN/dataset/dataset_download.sh +++ b/CNN/dataset/dataset_download.sh @@ -18,7 +18,7 @@ mkdir coco_2017 curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/coco_2017/val_dataset.py python3 val_dataset.py mv val_dataset.py ./coco_2017 -unzip -q coco2017val.zip -d ../model/yolo_v5/datasets && rm coco2017val.zip +unzip -q coco2017val.zip -d ../model/yolov5/datasets && rm coco2017val.zip #object detection video dataset From 9f80e8f83e4ac62c0012d4abd6ce12d8e499fd3a Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Fri, 17 Feb 2023 13:57:35 +0900 Subject: [PATCH 24/96] Update dataset_download.sh coco image data download path change --- CNN/dataset/dataset_download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/dataset/dataset_download.sh b/CNN/dataset/dataset_download.sh index c27f551..7b6b6e3 100644 --- a/CNN/dataset/dataset_download.sh +++ b/CNN/dataset/dataset_download.sh @@ -18,7 +18,7 @@ mkdir coco_2017 curl -O https://edge-inference.s3.us-west-2.amazonaws.com/CNN/dataset/coco_2017/val_dataset.py python3 val_dataset.py mv val_dataset.py ./coco_2017 -unzip -q coco2017val.zip -d ../model/yolov5/datasets && rm coco2017val.zip +unzip -q coco2017val.zip -d ./ && rm coco2017val.zip #object detection video dataset From 060951e7157701ac920dffe20b0eeb141e0de275 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 17 Feb 2023 15:01:45 +0900 Subject: [PATCH 25/96] =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=98=A4=EB=A5=98?= =?UTF-8?q?=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 367c2fe..5722b3a 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -76,12 +76,12 @@ def save_model(model, saved_model_dir): loaded_models = {} -model_path = 'mobilenet_saved_model' model_names = models_detail.keys() for model_name in model_names: + model_path = f'{model_name}_saved_model' if os.path.isdir(model_path) == False: print('model save') - save_model(model_name, model_name + '_saved_model') + save_model(model_name, model_path) loaded_models[model_name] = tf.keras.models.load_model(model_path) app = Flask(__name__) From 644287548e94338a3ac1f673d7a8aada4964748f Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 17 Feb 2023 15:24:42 +0900 Subject: [PATCH 26/96] =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=98=A4=EB=A5=98?= =?UTF-8?q?=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 5722b3a..1361a95 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -96,14 +96,14 @@ def mobilenetv1(): @app.route('/mobilenetv2') def mobilenetv2(): - result = loaded_models['mobilenet_v2'].predict(mobilenetv1_test_image_preprocessed) + result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) print(result) return 'mobilenetv2 inference success' @app.route('/inceptionv3') def inceptionv3(): - result = loaded_models['inception_v3'].predict(mobilenetv1_test_image_preprocessed) + result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) print(result) return 'inceptionv3 inference success' From 1b8b4961a5aa10927ed01ec5ff1e02d0e868293b Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 17 Feb 2023 16:26:06 +0900 Subject: [PATCH 27/96] =?UTF-8?q?=EC=B6=94=EB=A1=A0=20=EC=8B=9C=EA=B0=84?= =?UTF-8?q?=20=EC=B8=A1=EC=A0=95=EC=9D=84=20=EC=9C=84=ED=95=9C=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 1361a95..817eda3 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -3,6 +3,7 @@ import numpy as np import shutil import os +import time gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: @@ -89,23 +90,29 @@ def save_model(model, saved_model_dir): @app.route('/mobilenetv1') def mobilenetv1(): + inference_start_time = time.time() result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + inference_time = time.time() - inference_start_time print(result) - return 'mobilenetv1 inference success' + return f'mobilenetv1 inference success\ntime:{inference_time}\n' @app.route('/mobilenetv2') def mobilenetv2(): + inference_start_time = time.time() result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) + inference_time = time.time() - inference_start_time print(result) - return 'mobilenetv2 inference success' + return f'mobilenetv2 inference success\ntime:{inference_time}\n' @app.route('/inceptionv3') def inceptionv3(): + inference_start_time = time.time() result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) + inference_time = time.time() - inference_start_time print(result) - return 'inceptionv3 inference success' + return f'inceptionv3 inference success\ntime:{inference_time}\n' app.run(host='localhost', port=5001) From 117a6237a9a13334b9be5b1c643ee45faa3c7091 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 20 Feb 2023 15:33:46 +0900 Subject: [PATCH 28/96] =?UTF-8?q?tf=20record=20=EC=82=AC=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 77 +++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 817eda3..063e855 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -68,6 +68,71 @@ def image_preprocess(image_array, model_name): mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') +img_size = 224 + +def deserialize_image_record(record): + feature_map = {'image/encoded': tf.io.FixedLenFeature([], tf.string, ''), + 'image/class/label': tf.io.FixedLenFeature([1], tf.int64, -1), + 'image/class/text': tf.io.FixedLenFeature([], tf.string, '')} + obj = tf.io.parse_single_example(serialized=record, features=feature_map) + imgdata = obj['image/encoded'] + label = tf.cast(obj['image/class/label'], tf.int32) + label_text = tf.cast(obj['image/class/text'], tf.string) + return imgdata, label, label_text + + +def val_preprocessing(record): + imgdata, label, label_text = deserialize_image_record(record) + label -= 1 + image = tf.io.decode_jpeg(imgdata, channels=3, + fancy_upscaling=False, + dct_method='INTEGER_FAST') + + shape = tf.shape(image) + height = tf.cast(shape[0], tf.float32) + width = tf.cast(shape[1], tf.float32) + side = tf.cast(tf.convert_to_tensor(256, dtype=tf.int32), tf.float32) + + scale = tf.cond(tf.greater(height, width), + lambda: side / width, + lambda: side / height) + + new_height = tf.cast(tf.math.rint(height * scale), tf.int32) + new_width = tf.cast(tf.math.rint(width * scale), tf.int32) + + image = tf.image.resize(image, [new_height, new_width], method='bicubic') + image = tf.image.resize_with_crop_or_pad(image, img_size, img_size) + + image = tf.keras.applications.mobilenet.preprocess_input(image) + + return image, label, label_text + + +def get_dataset(batch_size, use_cache=False): + data_dir = './dataset/imagenet/imagenet_1000' + files = tf.io.gfile.glob(os.path.join(data_dir)) + dataset = tf.data.TFRecordDataset(files) + + dataset = dataset.map(map_func=val_preprocessing, num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size=batch_size) + dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) + dataset = dataset.repeat(count=1) + + if use_cache: + shutil.rmtree('tfdatacache', ignore_errors=True) + os.mkdir('tfdatacache') + dataset = dataset.cache(f'./tfdatacache/imagenet_val') + + return dataset + + +batchsize = 1 + +mobilenet_dataset = get_dataset(batchsize) + +img_size = 299 +inception_dataset = get_dataset(batchsize) + def save_model(model, saved_model_dir): model = models_detail[model] @@ -91,7 +156,9 @@ def save_model(model, saved_model_dir): @app.route('/mobilenetv1') def mobilenetv1(): inference_start_time = time.time() - result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + # result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + model = loaded_models['mobilenet'] + result = model(mobilenet_dataset) inference_time = time.time() - inference_start_time print(result) return f'mobilenetv1 inference success\ntime:{inference_time}\n' @@ -100,7 +167,9 @@ def mobilenetv1(): @app.route('/mobilenetv2') def mobilenetv2(): inference_start_time = time.time() - result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) + # result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) + model = loaded_models['mobilenet_v2'] + result = model(mobilenet_dataset) inference_time = time.time() - inference_start_time print(result) return f'mobilenetv2 inference success\ntime:{inference_time}\n' @@ -109,7 +178,9 @@ def mobilenetv2(): @app.route('/inceptionv3') def inceptionv3(): inference_start_time = time.time() - result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) + # result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) + model = loaded_models['inception_v3'] + result = model(inception_dataset) inference_time = time.time() - inference_start_time print(result) return f'inceptionv3 inference success\ntime:{inference_time}\n' From ea74c17b0c509ca15250995b90a1fbd6476d527f Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 20 Feb 2023 15:50:10 +0900 Subject: [PATCH 29/96] =?UTF-8?q?=EC=BD=94=EB=93=9C=EC=98=A4=EB=A5=98?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 063e855..2435fed 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -128,10 +128,10 @@ def get_dataset(batch_size, use_cache=False): batchsize = 1 -mobilenet_dataset = get_dataset(batchsize) +mobilenet_dataset, _, _ = get_dataset(batchsize)[0] img_size = 299 -inception_dataset = get_dataset(batchsize) +inception_dataset, _, _ = get_dataset(batchsize)[0] def save_model(model, saved_model_dir): From aab49ed105780aa451b7cda79a327629daa60403 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 20 Feb 2023 15:57:06 +0900 Subject: [PATCH 30/96] =?UTF-8?q?=EC=BD=94=EB=93=9C=EC=98=A4=EB=A5=98?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 2435fed..b4ad6cd 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -128,10 +128,11 @@ def get_dataset(batch_size, use_cache=False): batchsize = 1 -mobilenet_dataset, _, _ = get_dataset(batchsize)[0] +mobilenet_dataset = get_dataset(batchsize) +print(mobilenet_dataset) img_size = 299 -inception_dataset, _, _ = get_dataset(batchsize)[0] +inception_dataset = get_dataset(batchsize) def save_model(model, saved_model_dir): @@ -156,33 +157,38 @@ def save_model(model, saved_model_dir): @app.route('/mobilenetv1') def mobilenetv1(): inference_start_time = time.time() - # result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - model = loaded_models['mobilenet'] - result = model(mobilenet_dataset) + for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): + # result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + model = loaded_models['mobilenet'] + model(validation_ds) + break inference_time = time.time() - inference_start_time - print(result) + return f'mobilenetv1 inference success\ntime:{inference_time}\n' @app.route('/mobilenetv2') def mobilenetv2(): inference_start_time = time.time() - # result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) - model = loaded_models['mobilenet_v2'] - result = model(mobilenet_dataset) + for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): + # result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) + model = loaded_models['mobilenet_v2'] + model(validation_ds) + break inference_time = time.time() - inference_start_time - print(result) return f'mobilenetv2 inference success\ntime:{inference_time}\n' @app.route('/inceptionv3') def inceptionv3(): inference_start_time = time.time() - # result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) - model = loaded_models['inception_v3'] - result = model(inception_dataset) + for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): + # result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) + model = loaded_models['inception_v3'] + model(validation_ds) + break inference_time = time.time() - inference_start_time - print(result) + return f'inceptionv3 inference success\ntime:{inference_time}\n' From 90f3c00252a1589b7520a7f5c5d0b77c3378dbe9 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 20 Feb 2023 16:08:42 +0900 Subject: [PATCH 31/96] =?UTF-8?q?=EB=AA=A8=EB=8D=B8=20=EB=A1=9C=EB=93=9C?= =?UTF-8?q?=EC=9C=84=EC=B9=98=20=EB=B3=80=EA=B2=BD=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index b4ad6cd..9b819ff 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -153,14 +153,15 @@ def save_model(model, saved_model_dir): app = Flask(__name__) +mobilenetv1_model = loaded_models['mobilenet'] + @app.route('/mobilenetv1') def mobilenetv1(): inference_start_time = time.time() for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): # result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - model = loaded_models['mobilenet'] - model(validation_ds) + mobilenetv1_model(validation_ds) break inference_time = time.time() - inference_start_time @@ -188,7 +189,7 @@ def inceptionv3(): model(validation_ds) break inference_time = time.time() - inference_start_time - + return f'inceptionv3 inference success\ntime:{inference_time}\n' From 1a57e3b04a489244293c9a48892abe09de6afa08 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 20 Feb 2023 16:56:08 +0900 Subject: [PATCH 32/96] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 92 ++++------------------------------- 1 file changed, 10 insertions(+), 82 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 9b819ff..6dda200 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -68,72 +68,6 @@ def image_preprocess(image_array, model_name): mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') -img_size = 224 - -def deserialize_image_record(record): - feature_map = {'image/encoded': tf.io.FixedLenFeature([], tf.string, ''), - 'image/class/label': tf.io.FixedLenFeature([1], tf.int64, -1), - 'image/class/text': tf.io.FixedLenFeature([], tf.string, '')} - obj = tf.io.parse_single_example(serialized=record, features=feature_map) - imgdata = obj['image/encoded'] - label = tf.cast(obj['image/class/label'], tf.int32) - label_text = tf.cast(obj['image/class/text'], tf.string) - return imgdata, label, label_text - - -def val_preprocessing(record): - imgdata, label, label_text = deserialize_image_record(record) - label -= 1 - image = tf.io.decode_jpeg(imgdata, channels=3, - fancy_upscaling=False, - dct_method='INTEGER_FAST') - - shape = tf.shape(image) - height = tf.cast(shape[0], tf.float32) - width = tf.cast(shape[1], tf.float32) - side = tf.cast(tf.convert_to_tensor(256, dtype=tf.int32), tf.float32) - - scale = tf.cond(tf.greater(height, width), - lambda: side / width, - lambda: side / height) - - new_height = tf.cast(tf.math.rint(height * scale), tf.int32) - new_width = tf.cast(tf.math.rint(width * scale), tf.int32) - - image = tf.image.resize(image, [new_height, new_width], method='bicubic') - image = tf.image.resize_with_crop_or_pad(image, img_size, img_size) - - image = tf.keras.applications.mobilenet.preprocess_input(image) - - return image, label, label_text - - -def get_dataset(batch_size, use_cache=False): - data_dir = './dataset/imagenet/imagenet_1000' - files = tf.io.gfile.glob(os.path.join(data_dir)) - dataset = tf.data.TFRecordDataset(files) - - dataset = dataset.map(map_func=val_preprocessing, num_parallel_calls=tf.data.experimental.AUTOTUNE) - dataset = dataset.batch(batch_size=batch_size) - dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE) - dataset = dataset.repeat(count=1) - - if use_cache: - shutil.rmtree('tfdatacache', ignore_errors=True) - os.mkdir('tfdatacache') - dataset = dataset.cache(f'./tfdatacache/imagenet_val') - - return dataset - - -batchsize = 1 - -mobilenet_dataset = get_dataset(batchsize) -print(mobilenet_dataset) - -img_size = 299 -inception_dataset = get_dataset(batchsize) - def save_model(model, saved_model_dir): model = models_detail[model] @@ -153,43 +87,37 @@ def save_model(model, saved_model_dir): app = Flask(__name__) -mobilenetv1_model = loaded_models['mobilenet'] - @app.route('/mobilenetv1') def mobilenetv1(): inference_start_time = time.time() - for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): - # result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - mobilenetv1_model(validation_ds) - break + result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) inference_time = time.time() - inference_start_time + print(result) + return f'mobilenetv1 inference success\ntime:{inference_time}\n' @app.route('/mobilenetv2') def mobilenetv2(): inference_start_time = time.time() - for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): - # result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) - model = loaded_models['mobilenet_v2'] - model(validation_ds) - break + result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) inference_time = time.time() - inference_start_time + + print(result) + return f'mobilenetv2 inference success\ntime:{inference_time}\n' @app.route('/inceptionv3') def inceptionv3(): inference_start_time = time.time() - for i, (validation_ds, batch_labels, _) in enumerate(mobilenet_dataset): - # result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) - model = loaded_models['inception_v3'] - model(validation_ds) - break + result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) inference_time = time.time() - inference_start_time + print(result) + return f'inceptionv3 inference success\ntime:{inference_time}\n' From e6efc368a5c8c5d0b8b67670906473fa7ce5246d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 21 Feb 2023 14:40:28 +0900 Subject: [PATCH 33/96] =?UTF-8?q?edge=20=EC=B6=94=EB=A1=A0=20=EC=84=9C?= =?UTF-8?q?=EB=B2=84=20=EC=8B=A4=ED=96=89=EC=8B=9C=20=EB=A1=9C=EB=93=9C?= =?UTF-8?q?=ED=95=A0=20=EB=AA=A8=EB=8D=B8=EC=9D=84=20=EC=9D=B8=EC=9E=90?= =?UTF-8?q?=EB=A1=9C=20=EB=B0=9B=EC=9D=84=EC=88=98=20=EC=9E=88=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 6dda200..1954c0c 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -1,3 +1,5 @@ +import argparse + from flask import Flask import tensorflow as tf import numpy as np @@ -18,6 +20,11 @@ inception_v3 ) +parser = argparse.ArgumentParser() +parser.add_argument('--model', default='all', type=str) +args = parser.parse_args() +models_to_load = args.model + models = { 'mobilenet': mobilenet, 'mobilenet_v2': mobilenet_v2, @@ -77,13 +84,21 @@ def save_model(model, saved_model_dir): loaded_models = {} -model_names = models_detail.keys() -for model_name in model_names: - model_path = f'{model_name}_saved_model' +if models_to_load == 'all': + model_names = models_detail.keys() + for model_name in model_names: + model_path = f'{model_name}_saved_model' + if os.path.isdir(model_path) == False: + print('model save') + save_model(model_name, model_path) + loaded_models[model_name] = tf.keras.models.load_model(model_path) +elif models_to_load in models_detail.keys(): + model_path = f'{models_to_load}_saved_model' if os.path.isdir(model_path) == False: print('model save') - save_model(model_name, model_path) - loaded_models[model_name] = tf.keras.models.load_model(model_path) + save_model(models_to_load, model_path) + loaded_models[models_to_load] = tf.keras.models.load_model(model_path) + app = Flask(__name__) From 91756c5aa09f8982f0b0b323fafbb1df8c7b31b4 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 21 Feb 2023 15:05:04 +0900 Subject: [PATCH 34/96] =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EB=A6=AC=ED=8E=99?= =?UTF-8?q?=ED=86=A0=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index 1954c0c..f47a7aa 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -21,9 +21,9 @@ ) parser = argparse.ArgumentParser() -parser.add_argument('--model', default='all', type=str) +parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) args = parser.parse_args() -models_to_load = args.model +models_to_load = args.model.split(',') models = { 'mobilenet': mobilenet, @@ -84,21 +84,14 @@ def save_model(model, saved_model_dir): loaded_models = {} -if models_to_load == 'all': +for model_name in models_to_load: model_names = models_detail.keys() - for model_name in model_names: + if model_name in model_names: model_path = f'{model_name}_saved_model' if os.path.isdir(model_path) == False: print('model save') save_model(model_name, model_path) loaded_models[model_name] = tf.keras.models.load_model(model_path) -elif models_to_load in models_detail.keys(): - model_path = f'{models_to_load}_saved_model' - if os.path.isdir(model_path) == False: - print('model save') - save_model(models_to_load, model_path) - loaded_models[models_to_load] = tf.keras.models.load_model(model_path) - app = Flask(__name__) From cd003483ab26fb3e433127ad451f0a97eca58c37 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 23 Feb 2023 17:20:36 +0900 Subject: [PATCH 35/96] =?UTF-8?q?=EC=84=9C=EB=B2=84=20=EC=8B=A4=ED=96=89?= =?UTF-8?q?=EC=8B=9C=20=ED=98=B8=EC=8A=A4=ED=8A=B8=EC=9D=B4=EB=A6=84?= =?UTF-8?q?=EA=B3=BC=20=ED=8F=AC=ED=8A=B8=EB=B2=88=ED=98=B8=EB=A5=BC=20?= =?UTF-8?q?=EC=9E=85=EB=A0=A5=20=EB=B0=9B=EC=9D=84=20=EC=88=98=20=EC=9E=88?= =?UTF-8?q?=EA=B2=8C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request.py index f47a7aa..3ffbc06 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request.py @@ -22,8 +22,12 @@ parser = argparse.ArgumentParser() parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +parser.add_argument('--hostname', default='localhost', type=str) +parser.add_argument('--port', default=5001, type=int) args = parser.parse_args() models_to_load = args.model.split(',') +hostname = args.hostname +port = args.port models = { 'mobilenet': mobilenet, @@ -92,6 +96,9 @@ def save_model(model, saved_model_dir): print('model save') save_model(model_name, model_path) loaded_models[model_name] = tf.keras.models.load_model(model_path) + else: + print(f'model names must be in {model_names}') + exit(1) app = Flask(__name__) @@ -129,4 +136,4 @@ def inceptionv3(): return f'inceptionv3 inference success\ntime:{inference_time}\n' -app.run(host='localhost', port=5001) +app.run(host=hostname, port=port) From 6cfc9b58cc4796fbc647a96e5afe0fb68a8d6c40 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 23 Feb 2023 17:58:18 +0900 Subject: [PATCH 36/96] =?UTF-8?q?edge=5Finferece=5Freqest=EB=A5=BC=20edge?= =?UTF-8?q?=5Finference=5Fserver=EB=A1=9C=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ge_inference_request.py => edge_inference_request_server.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename CNN/{edge_inference_request.py => edge_inference_request_server.py} (98%) diff --git a/CNN/edge_inference_request.py b/CNN/edge_inference_request_server.py similarity index 98% rename from CNN/edge_inference_request.py rename to CNN/edge_inference_request_server.py index 3ffbc06..2fac096 100644 --- a/CNN/edge_inference_request.py +++ b/CNN/edge_inference_request_server.py @@ -22,7 +22,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) -parser.add_argument('--hostname', default='localhost', type=str) +parser.add_argument('--hostname', default='0.0.0.0', type=str) parser.add_argument('--port', default=5001, type=int) args = parser.parse_args() models_to_load = args.model.split(',') From 9f085e4206834e4953ae07132fbd98c71137a916 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 23 Feb 2023 20:25:06 +0900 Subject: [PATCH 37/96] =?UTF-8?q?=EC=97=A3=EC=A7=80=20=EC=B6=94=EB=A1=A0?= =?UTF-8?q?=EC=84=9C=EB=B2=84=EC=97=90=20=EC=B6=94=EB=A1=A0=EC=9D=84=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=ED=95=98=EB=8A=94=20=EA=B0=84=EB=8B=A8?= =?UTF-8?q?=ED=95=9C=20=EC=8A=A4=EC=BC=80=EC=A4=84=EB=9F=AC=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 27 +++++++++++++++++++++++++ CNN/edge_inference_request_server.py | 6 +++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 CNN/edge_inference_request_scheduler.py diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py new file mode 100644 index 0000000..3e840f8 --- /dev/null +++ b/CNN/edge_inference_request_scheduler.py @@ -0,0 +1,27 @@ +import argparse +import requests + +parser = argparse.ArgumentParser() +parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +parser.add_argument('--hostname', required=True, type=str) +parser.add_argument('--port', required=True, type=int) + +args = parser.parse_args() + +models_to_inference = args.model.split(',') +hostname = args.hostname +port = args.port + +inference_request_url = f'http://{hostname}:{port}/' + + +def model_reqest(model): + url = inference_request_url + model + res = requests.post(url) + + return res.text + + +for model in models_to_inference: + res = model_reqest(model) + print(res) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 2fac096..9cb85b1 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -103,7 +103,7 @@ def save_model(model, saved_model_dir): app = Flask(__name__) -@app.route('/mobilenetv1') +@app.route('/mobilenet') def mobilenetv1(): inference_start_time = time.time() result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) @@ -114,7 +114,7 @@ def mobilenetv1(): return f'mobilenetv1 inference success\ntime:{inference_time}\n' -@app.route('/mobilenetv2') +@app.route('/mobilenet_v2') def mobilenetv2(): inference_start_time = time.time() result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) @@ -125,7 +125,7 @@ def mobilenetv2(): return f'mobilenetv2 inference success\ntime:{inference_time}\n' -@app.route('/inceptionv3') +@app.route('/inception_v3') def inceptionv3(): inference_start_time = time.time() result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) From 73b9b43bf53f265d07a1796da1f3aa877d44d1b8 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 23 Feb 2023 20:36:51 +0900 Subject: [PATCH 38/96] =?UTF-8?q?=EC=9A=94=EC=B2=AD=ED=95=A0=EB=95=8C=20po?= =?UTF-8?q?st=20=EB=8C=80=EC=8B=A0=20get=20=EC=82=AC=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 3e840f8..391271a 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -17,7 +17,7 @@ def model_reqest(model): url = inference_request_url + model - res = requests.post(url) + res = requests.get(url) return res.text From 4797ec2944c16d22b3e88b4746b7d556882f2e6c Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 12:41:53 +0900 Subject: [PATCH 39/96] =?UTF-8?q?poisson=EC=9D=84=20=EC=9D=B4=EC=9A=A9?= =?UTF-8?q?=ED=95=98=EC=97=AC=20request=EB=A5=BC=20=EB=B3=B4=EB=82=B4?= =?UTF-8?q?=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 32 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 391271a..e304638 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -1,5 +1,8 @@ import argparse import requests +import roundrobin +from numpy import random +import time parser = argparse.ArgumentParser() parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) @@ -15,13 +18,34 @@ inference_request_url = f'http://{hostname}:{port}/' -def model_reqest(model): +def model_request(model): url = inference_request_url + model res = requests.get(url) return res.text -for model in models_to_inference: - res = model_reqest(model) - print(res) +models = [('mobilenet', 1), ('mobilenet_v2', 3), ('inception_v3', 6)] +get_weighted_smooth = roundrobin.smooth(models) + +events_avg = 10 +total_event_num = 10 + +poisson_distribution = random.poisson(events_avg, total_event_num) + +for event_num in poisson_distribution: + print('request count: ', event_num) + + for idx in event_num: + current_inference_model = get_weighted_smooth() + + request_start_time = time.time() + res = model_request(current_inference_model) + print('total request time: ', time.time() - request_start_time) + + print(res) + + + + + From 60a88f8ab4cad34c62b5af118a9881690caee89e Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 15:05:40 +0900 Subject: [PATCH 40/96] =?UTF-8?q?=EC=84=9C=EB=B2=84=EC=9D=98=20=ED=98=84?= =?UTF-8?q?=EC=9E=AC=20=EC=A7=84=ED=96=89=EC=83=81=ED=99=A9=EC=9D=84=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD=ED=95=98=EA=B3=A0=20=EC=97=90=EB=9F=AC=EB=A5=BC=20?= =?UTF-8?q?=EC=88=98=EC=A0=95=ED=95=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 2 +- CNN/edge_inference_request_server.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index e304638..b613e0d 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -31,7 +31,7 @@ def model_request(model): events_avg = 10 total_event_num = 10 -poisson_distribution = random.poisson(events_avg, total_event_num) +poisson_distribution = list(random.poisson(events_avg, total_event_num)) for event_num in poisson_distribution: print('request count: ', event_num) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 9cb85b1..458452a 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -62,6 +62,7 @@ def image_preprocess(image_array, model_name): return models[model_name].preprocess_input( image_array[tf.newaxis, ...]) +print('\npreprossing images...') mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' @@ -79,12 +80,14 @@ def image_preprocess(image_array, model_name): mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') +print('image preprocessing completed!\n') def save_model(model, saved_model_dir): model = models_detail[model] shutil.rmtree(saved_model_dir, ignore_errors=True) model.save(saved_model_dir, include_optimizer=False, save_format='tf') +print('\nsaving and loading models...') loaded_models = {} @@ -100,6 +103,8 @@ def save_model(model, saved_model_dir): print(f'model names must be in {model_names}') exit(1) +print('saving and loading models completed!\n') + app = Flask(__name__) From b5bc69e42b0d0e65e0e395b308660688846e3bf7 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 15:06:56 +0900 Subject: [PATCH 41/96] =?UTF-8?q?error=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index b613e0d..3be7a54 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -31,12 +31,12 @@ def model_request(model): events_avg = 10 total_event_num = 10 -poisson_distribution = list(random.poisson(events_avg, total_event_num)) +poisson_distribution = random.poisson(events_avg, total_event_num) for event_num in poisson_distribution: print('request count: ', event_num) - for idx in event_num: + for idx in range(event_num): current_inference_model = get_weighted_smooth() request_start_time = time.time() From 1d802b606baa2211f7b4b843affd5633e4f6ef45 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:04:38 +0900 Subject: [PATCH 42/96] =?UTF-8?q?=EC=B6=94=EB=A1=A0=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=EC=9D=84=20=EB=B3=91=EB=A0=AC=EB=A1=9C=20=EB=B3=B4=EB=82=BC=20?= =?UTF-8?q?=EC=88=98=20=EC=9E=88=EA=B2=8C=20=EC=93=B0=EB=A0=88=EB=93=9C=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 52 ++++++++++++++++--------- CNN/edge_inference_request_server.py | 6 +-- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 3be7a54..9495d55 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -3,6 +3,7 @@ import roundrobin from numpy import random import time +from threading import Thread parser = argparse.ArgumentParser() parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) @@ -19,31 +20,46 @@ def model_request(model): + req_processing_start_time = time.time() url = inference_request_url + model res = requests.get(url) + print('total request time: ', time.time() - req_processing_start_time) + print(res.text) - return res.text + return +request_num = 10 models = [('mobilenet', 1), ('mobilenet_v2', 3), ('inception_v3', 6)] get_weighted_smooth = roundrobin.smooth(models) - -events_avg = 10 -total_event_num = 10 - -poisson_distribution = random.poisson(events_avg, total_event_num) - -for event_num in poisson_distribution: - print('request count: ', event_num) - - for idx in range(event_num): - current_inference_model = get_weighted_smooth() - - request_start_time = time.time() - res = model_request(current_inference_model) - print('total request time: ', time.time() - request_start_time) - - print(res) +requests_list = [models[0][0] for _ in range(request_num)] + +threads = [] +for req in requests_list: + th = Thread(target=model_request, args=req) + th.start() + threads.append(th) + +for th in threads: + th.join() + + +# events_avg = 10 +# total_event_num = 10 +# +# poisson_distribution = random.poisson(events_avg, total_event_num) +# +# for event_num in poisson_distribution: +# print('request count: ', event_num) +# +# for idx in range(event_num): +# current_inference_model = get_weighted_smooth() +# +# request_start_time = time.time() +# res = model_request(current_inference_model) +# print('total request time: ', time.time() - request_start_time) +# +# print(res) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 458452a..004a3f0 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -116,7 +116,7 @@ def mobilenetv1(): print(result) - return f'mobilenetv1 inference success\ntime:{inference_time}\n' + return f'mobilenetv1 inference success\ninference time:{inference_time}\n' @app.route('/mobilenet_v2') @@ -127,7 +127,7 @@ def mobilenetv2(): print(result) - return f'mobilenetv2 inference success\ntime:{inference_time}\n' + return f'mobilenetv2 inference success\ninference time:{inference_time}\n' @app.route('/inception_v3') @@ -138,7 +138,7 @@ def inceptionv3(): print(result) - return f'inceptionv3 inference success\ntime:{inference_time}\n' + return f'inceptionv3 inference success\ninference time:{inference_time}\n' app.run(host=hostname, port=port) From 86a3455123144491c513bf422c8c90b8ad3fa033 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:16:41 +0900 Subject: [PATCH 43/96] error fix --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 9495d55..0436a2a 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -36,7 +36,7 @@ def model_request(model): threads = [] for req in requests_list: - th = Thread(target=model_request, args=req) + th = Thread(target=model_request, args=(req)) th.start() threads.append(th) From 4a4f8927c863a37547c17f2841fc37578553d17b Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:19:06 +0900 Subject: [PATCH 44/96] error fix --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 0436a2a..d1c80b2 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -36,7 +36,7 @@ def model_request(model): threads = [] for req in requests_list: - th = Thread(target=model_request, args=(req)) + th = Thread(target=model_request, args=(req,)) th.start() threads.append(th) From 77d47666c8fb23ba712b0dd2109b524d40c69466 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:21:47 +0900 Subject: [PATCH 45/96] =?UTF-8?q?=EC=93=B0=EB=A0=88=EB=93=9C=EA=B0=80=20?= =?UTF-8?q?=EC=A0=95=EC=83=81=20=EC=9E=91=EB=8F=99=ED=95=98=EB=8A=94?= =?UTF-8?q?=EC=A7=80=20=ED=99=95=EC=9D=B8=ED=95=98=EA=B8=B0=20=EC=9C=84?= =?UTF-8?q?=ED=95=9C=20=EC=BD=94=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index d1c80b2..b4ff4ff 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -19,11 +19,11 @@ inference_request_url = f'http://{hostname}:{port}/' -def model_request(model): +def model_request(model, order): req_processing_start_time = time.time() url = inference_request_url + model res = requests.get(url) - print('total request time: ', time.time() - req_processing_start_time) + print(f'[{order}] total request time: ', time.time() - req_processing_start_time) print(res.text) return @@ -35,8 +35,10 @@ def model_request(model): requests_list = [models[0][0] for _ in range(request_num)] threads = [] +order = 0 for req in requests_list: - th = Thread(target=model_request, args=(req,)) + order += 1 + th = Thread(target=model_request, args=(req, order)) th.start() threads.append(th) From 4ad33ad7a2c7c7b608fa436692f8f4c53c24a138 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:24:34 +0900 Subject: [PATCH 46/96] =?UTF-8?q?=EC=B6=9C=EB=A0=A5=20=EB=A9=94=EC=8B=9C?= =?UTF-8?q?=EC=A7=80=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index b4ff4ff..d25f7a3 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -23,8 +23,8 @@ def model_request(model, order): req_processing_start_time = time.time() url = inference_request_url + model res = requests.get(url) - print(f'[{order}] total request time: ', time.time() - req_processing_start_time) - print(res.text) + processing_time = time.time() - req_processing_start_time + print(f'[{order}] total request time: {processing_time}\n{res.text}') return From b2d8bf0819231d012c79ce0dd2b1de91df43b52e Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:34:53 +0900 Subject: [PATCH 47/96] =?UTF-8?q?flask=20=EC=84=9C=EB=B2=84=EA=B0=80=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=EC=9D=84=20=EB=B3=91=EB=A0=AC=EC=A0=81?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EC=8B=A4=ED=96=89=ED=95=98=EB=8A=94?= =?UTF-8?q?=EC=A7=80=20=ED=99=95=EC=9D=B8=ED=95=98=EB=8A=94=20=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 004a3f0..fcc9ae3 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -107,14 +107,18 @@ def save_model(model, saved_model_dir): app = Flask(__name__) +order = 0 @app.route('/mobilenet') def mobilenetv1(): + global order + order += 1 + print(f'[{order}] inference start') inference_start_time = time.time() result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) inference_time = time.time() - inference_start_time - - print(result) + print(f'[{order}] inference end') + # print(result) return f'mobilenetv1 inference success\ninference time:{inference_time}\n' From 140ff0aec1d43573916b1e0a81d9c5b74aba56df Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 16:49:04 +0900 Subject: [PATCH 48/96] =?UTF-8?q?=EC=B6=9C=EB=A0=A5=EB=A9=94=EC=8B=9C?= =?UTF-8?q?=EC=A7=80=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index fcc9ae3..53c2473 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -113,11 +113,12 @@ def save_model(model, saved_model_dir): def mobilenetv1(): global order order += 1 - print(f'[{order}] inference start') + my_order = order + print(f'[{my_order}] inference start') inference_start_time = time.time() result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) inference_time = time.time() - inference_start_time - print(f'[{order}] inference end') + print(f'[{my_order}] inference end') # print(result) return f'mobilenetv1 inference success\ninference time:{inference_time}\n' From 9432e1046276dbd0332ddcdd1b2d91f6aeb0f48c Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 17:11:22 +0900 Subject: [PATCH 49/96] =?UTF-8?q?=EC=84=9C=EB=B2=84=EC=97=90=EC=84=9C=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=EA=B2=B0=EA=B3=BC=20=EC=B6=9C=EB=A0=A5?= =?UTF-8?q?=EC=8B=9C=20=EC=99=84=EB=A3=8C=EC=8B=9C=EA=B0=84=EB=8F=84=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 53c2473..741d07e 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -114,14 +114,14 @@ def mobilenetv1(): global order order += 1 my_order = order - print(f'[{my_order}] inference start') + print(f'[{my_order}] inference start: {time.time()}') inference_start_time = time.time() result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) inference_time = time.time() - inference_start_time - print(f'[{my_order}] inference end') + print(f'[{my_order}] inference end: {time.time()}') # print(result) - return f'mobilenetv1 inference success\ninference time:{inference_time}\n' + return f'mobilenetv1 inference success\ninference time:{inference_time}\nend time:{time.time()}' @app.route('/mobilenet_v2') From c5ff1fb804672f88151c9f89591bee9794e7b37d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 24 Feb 2023 18:04:47 +0900 Subject: [PATCH 50/96] =?UTF-8?q?=EC=97=AC=EB=9F=AC=20=EC=9E=A5=EB=B9=84?= =?UTF-8?q?=EC=97=90=20=EC=9A=94=EC=B2=AD=EC=9D=84=20=EB=B3=B4=EB=82=BC=20?= =?UTF-8?q?=EC=88=98=20=EC=9E=88=EB=8A=94=20=EA=B5=AC=EC=A1=B0=EB=A1=9C=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index d25f7a3..33c26aa 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -6,22 +6,28 @@ from threading import Thread parser = argparse.ArgumentParser() -parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) -parser.add_argument('--hostname', required=True, type=str) +#parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +#parser.add_argument('--hostname', required=True, type=str) parser.add_argument('--port', required=True, type=int) args = parser.parse_args() -models_to_inference = args.model.split(',') -hostname = args.hostname +#models_to_inference = args.model.split(',') +#hostname = args.hostname port = args.port -inference_request_url = f'http://{hostname}:{port}/' +#inference_request_url = f'http://{hostname}:{port}/' -def model_request(model, order): +edge_url_info = {'nvidia-xavier2': f'http://192.168.0.30:{port}/', + 'nvidia-nano1': f'http://192.168.0.29:{port}/', + 'nvidia-tx2': f'http://192.168.0.8:{port}/' + } + +def model_request(edge, model, order): req_processing_start_time = time.time() - url = inference_request_url + model + #url = inference_request_url + model + url = edge_url_info.get(edge) + model res = requests.get(url) processing_time = time.time() - req_processing_start_time print(f'[{order}] total request time: {processing_time}\n{res.text}') @@ -38,7 +44,7 @@ def model_request(model, order): order = 0 for req in requests_list: order += 1 - th = Thread(target=model_request, args=(req, order)) + th = Thread(target=model_request, args=('nvidia-tx2', req, order)) th.start() threads.append(th) From 9294b92d0b5f159fd1064200959c499aaf96fcaf Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 26 Feb 2023 21:29:04 +0900 Subject: [PATCH 51/96] =?UTF-8?q?=EC=84=B8=20=EC=97=A3=EC=A7=80=EC=9E=A5?= =?UTF-8?q?=EB=B9=84=EC=97=90=20=EB=9D=BC=EC=9A=B4=EB=93=9C=EB=A1=9C?= =?UTF-8?q?=EB=B9=88=EC=9C=BC=EB=A1=9C=20=EC=B6=94=EB=A1=A0=EC=9D=84=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=ED=95=98=EB=8A=94=20=EC=8A=A4=EC=BC=80?= =?UTF-8?q?=EC=A4=84=EB=9F=AC=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 53 ++++++++++++++----------- CNN/edge_inference_request_server.py | 32 +++++++++------ 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 33c26aa..142e929 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -6,28 +6,28 @@ from threading import Thread parser = argparse.ArgumentParser() -#parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) -#parser.add_argument('--hostname', required=True, type=str) parser.add_argument('--port', required=True, type=int) args = parser.parse_args() -#models_to_inference = args.model.split(',') -#hostname = args.hostname port = args.port -#inference_request_url = f'http://{hostname}:{port}/' +edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', + 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] + }, + 'nvidia-tx2': {'url': f'http://192.168.0.8:{port}/', + 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] + }, + 'nvidia-nano1': {'url': f'http://192.168.0.29:{port}/', + 'model': ['mobilenet'] + } + } -edge_url_info = {'nvidia-xavier2': f'http://192.168.0.30:{port}/', - 'nvidia-nano1': f'http://192.168.0.29:{port}/', - 'nvidia-tx2': f'http://192.168.0.8:{port}/' - } - def model_request(edge, model, order): req_processing_start_time = time.time() - #url = inference_request_url + model - url = edge_url_info.get(edge) + model + edge_info = edges_info.get(edge) + url = edge_info.get('url') + model res = requests.get(url) processing_time = time.time() - req_processing_start_time print(f'[{order}] total request time: {processing_time}\n{res.text}') @@ -35,23 +35,35 @@ def model_request(edge, model, order): return -request_num = 10 -models = [('mobilenet', 1), ('mobilenet_v2', 3), ('inception_v3', 6)] -get_weighted_smooth = roundrobin.smooth(models) -requests_list = [models[0][0] for _ in range(request_num)] +model_edge_info = {} + +for edge_info_key in edges_info.keys(): + edge_info = edges_info.get(edge_info_key) + for model in edge_info.get('model'): + if model not in model_edge_info.keys(): + model_edge_info[model] = [] + + model_edge_info[model].append((edge_info_key, 1)) + +for i in model_edge_info.keys(): + model_info = model_edge_info.get(i) + model_edge_info[i] = roundrobin.smooth(model_info) + + +requests_list = ['mobilenet', 'mobilenet_v2', 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet'] threads = [] order = 0 for req in requests_list: + edge_to_inference = model_edge_info.get(req)() order += 1 - th = Thread(target=model_request, args=('nvidia-tx2', req, order)) + th = Thread(target=model_request, args=(edge_to_inference, req, order)) th.start() threads.append(th) for th in threads: th.join() - # events_avg = 10 # total_event_num = 10 # @@ -68,8 +80,3 @@ def model_request(edge, model, order): # print('total request time: ', time.time() - request_start_time) # # print(res) - - - - - diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 741d07e..8f3215c 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -107,30 +107,34 @@ def save_model(model, saved_model_dir): app = Flask(__name__) -order = 0 +# order = 0 + @app.route('/mobilenet') def mobilenetv1(): - global order - order += 1 - my_order = order - print(f'[{my_order}] inference start: {time.time()}') + # global order + # order += 1 + # my_order = order inference_start_time = time.time() + # print(f'[{my_order}] inference start: {inference_start_time}') result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - inference_time = time.time() - inference_start_time - print(f'[{my_order}] inference end: {time.time()}') - # print(result) + inference_end_time = time.time() + # print(f'[{my_order}] inference end: {inference_end_time}') + + inference_time = inference_end_time - inference_start_time - return f'mobilenetv1 inference success\ninference time:{inference_time}\nend time:{time.time()}' + return f'mobilenetv1 inference success\ninference time:{inference_time}\n' @app.route('/mobilenet_v2') def mobilenetv2(): inference_start_time = time.time() result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) - inference_time = time.time() - inference_start_time + inference_end_time = time.time() - print(result) + inference_time = inference_end_time - inference_start_time + + # print(result) return f'mobilenetv2 inference success\ninference time:{inference_time}\n' @@ -139,9 +143,11 @@ def mobilenetv2(): def inceptionv3(): inference_start_time = time.time() result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) - inference_time = time.time() - inference_start_time + inference_end_time = time.time() + + inference_time = inference_end_time - inference_start_time - print(result) + # print(result) return f'inceptionv3 inference success\ninference time:{inference_time}\n' From 12ec1ea0e29bfc816d6dd0c1836f457df98666fe Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 26 Feb 2023 21:43:44 +0900 Subject: [PATCH 52/96] =?UTF-8?q?=EB=AA=A8=EB=8D=B8=EC=9A=94=EC=B2=AD=20?= =?UTF-8?q?=EC=A6=9D=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 142e929..aaac67d 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -50,14 +50,16 @@ def model_request(edge, model, order): model_edge_info[i] = roundrobin.smooth(model_info) -requests_list = ['mobilenet', 'mobilenet_v2', 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet'] +requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet' + 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet' + 'inception_v3', 'mobilenet_v2', 'inception_v3'] threads = [] order = 0 for req in requests_list: edge_to_inference = model_edge_info.get(req)() order += 1 - th = Thread(target=model_request, args=(edge_to_inference, req, order)) + th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) th.start() threads.append(th) From 1a07c3feb3334518c2d30b6e41e255eaaace6266 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 26 Feb 2023 21:45:10 +0900 Subject: [PATCH 53/96] error fix --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index aaac67d..586c00f 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -50,8 +50,8 @@ def model_request(edge, model, order): model_edge_info[i] = roundrobin.smooth(model_info) -requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet' - 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet' +requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet', + 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet', 'inception_v3', 'mobilenet_v2', 'inception_v3'] threads = [] From d2f440f05182cc5972a722f8d38fc82cf3ba03e2 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 11:38:40 +0900 Subject: [PATCH 54/96] =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=A4=84=EB=9F=AC=20?= =?UTF-8?q?=EC=8B=A4=ED=96=89=EC=8B=9C=20=EC=9A=94=EC=B2=AD=EC=9D=84=20?= =?UTF-8?q?=EC=A0=84=EB=8B=AC=ED=95=A0=20=EC=9E=A5=EB=B9=84=EB=93=A4?= =?UTF-8?q?=EC=9D=84=20=EC=9D=B8=EC=9E=90=EB=A1=9C=20=EB=B0=9B=EC=9D=84=20?= =?UTF-8?q?=EC=88=98=20=EC=9E=88=EA=B2=8C=20=EC=88=98=EC=A0=95,=20?= =?UTF-8?q?=EC=A3=BC=EC=84=9D=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 64 +++++++++++++++++-------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 586c00f..7302575 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -6,12 +6,16 @@ from threading import Thread parser = argparse.ArgumentParser() -parser.add_argument('--port', required=True, type=int) +parser.add_argument('--edge', default=None, type=str) +parser.add_argument('--port', default=5001, type=int) args = parser.parse_args() +edges_to_inference = args.edge port = args.port + +# 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, @@ -24,7 +28,39 @@ } +# --edge 옵션이 없을 시 등록되어 있는 모든 장비들에 추론 요청, 요청장비들은 edges_info에 등록되어 있어야함. 입력 형식은 'a, b, ...' +edges_register = list(edges_info.keys()) + +if edges_to_inference is None: + edges_to_inference = edges_register +else: + edges_to_inference = edges_to_inference.split(',') + +for edge in edges_to_inference: + if edge not in edges_register: + print(f'--edge arg must be in {edges_register}') + exit(1) + + +# edges_info에 등록된 모델들 +models_register = [] + +for edge_info in edges_info.values(): + models_register.extend(edge_info.get('model')) + +models_register = set(models_register) + + +# 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 def model_request(edge, model, order): + if edge not in edges_register: + print(f'edge must be in {edges_register}') + return + + if model not in models_register: + print(f'model must be in {models_register}') + return + req_processing_start_time = time.time() edge_info = edges_info.get(edge) url = edge_info.get('url') + model @@ -35,6 +71,8 @@ def model_request(edge, model, order): return +# 현재 스케줄링 방식: 딕셔너리에 모델별로 엣지장비이름 등록, 들어오는 요청에 따라 각 장비들에 라운드로빈으로 스케줄링 +# 문제점 각 모델이 하나씩 들어오면 장비들 중 하나에만 요청이 들어감 -> 고민 model_edge_info = {} for edge_info_key in edges_info.keys(): @@ -45,11 +83,12 @@ def model_request(edge, model, order): model_edge_info[model].append((edge_info_key, 1)) -for i in model_edge_info.keys(): - model_info = model_edge_info.get(i) - model_edge_info[i] = roundrobin.smooth(model_info) +for model in model_edge_info.keys(): + dataset = model_edge_info.get(model) + model_edge_info[model] = roundrobin.smooth(dataset) +# 들어오는 요청들 임시 코드임!!! requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet', 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet', 'inception_v3', 'mobilenet_v2', 'inception_v3'] @@ -65,20 +104,3 @@ def model_request(edge, model, order): for th in threads: th.join() - -# events_avg = 10 -# total_event_num = 10 -# -# poisson_distribution = random.poisson(events_avg, total_event_num) -# -# for event_num in poisson_distribution: -# print('request count: ', event_num) -# -# for idx in range(event_num): -# current_inference_model = get_weighted_smooth() -# -# request_start_time = time.time() -# res = model_request(current_inference_model) -# print('total request time: ', time.time() - request_start_time) -# -# print(res) From 20c1a05f5372658ef695dc6604e0b19b80a3d6a1 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 13:50:03 +0900 Subject: [PATCH 55/96] =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 7302575..f32b91d 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -42,23 +42,25 @@ exit(1) -# edges_info에 등록된 모델들 -models_register = [] +# 추론 요청 할 장비들에서 요청 가능한 모델들 +models_to_inference = [] -for edge_info in edges_info.values(): - models_register.extend(edge_info.get('model')) +for edge_name in edges_to_inference: + edge_info = edges_info.get(edge_name) + models = edge_info.get('model') + models_to_inference.extend(models) -models_register = set(models_register) +models_to_inference = set(models_to_inference) # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 def model_request(edge, model, order): - if edge not in edges_register: - print(f'edge must be in {edges_register}') + if edge not in edges_to_inference: + print(f'edge must be in {edges_to_inference}') return - if model not in models_register: - print(f'model must be in {models_register}') + if model not in models_to_inference: + print(f'model must be in {models_to_inference}') return req_processing_start_time = time.time() @@ -67,7 +69,6 @@ def model_request(edge, model, order): res = requests.get(url) processing_time = time.time() - req_processing_start_time print(f'[{order}] total request time: {processing_time}\n{res.text}') - return From b6cb0680cc846d8c4b0e1fb4df25e87cce42817d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 14:06:49 +0900 Subject: [PATCH 56/96] =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index f32b91d..c3888bc 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -76,13 +76,21 @@ def model_request(edge, model, order): # 문제점 각 모델이 하나씩 들어오면 장비들 중 하나에만 요청이 들어감 -> 고민 model_edge_info = {} -for edge_info_key in edges_info.keys(): - edge_info = edges_info.get(edge_info_key) +for edge in edges_to_inference: + edge_info = edges_info.get(edge) for model in edge_info.get('model'): if model not in model_edge_info.keys(): model_edge_info[model] = [] - model_edge_info[model].append((edge_info_key, 1)) + model_edge_info[model].append((edge, 1)) + +# for edge_info_key in edges_info.keys(): +# edge_info = edges_info.get(edge_info_key) +# for model in edge_info.get('model'): +# if model not in model_edge_info.keys(): +# model_edge_info[model] = [] +# +# model_edge_info[model].append((edge_info_key, 1)) for model in model_edge_info.keys(): dataset = model_edge_info.get(model) From d2216ca79f2b5c6e1d76fc0e9c3582175d29bf4e Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 14:14:46 +0900 Subject: [PATCH 57/96] =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index c3888bc..4eae0e1 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -105,7 +105,10 @@ def model_request(edge, model, order): threads = [] order = 0 for req in requests_list: - edge_to_inference = model_edge_info.get(req)() + edge_to_inference = model_edge_info.get(req) + if edges_to_inference is not None: + edges_to_inference = edges_to_inference() + order += 1 th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) th.start() From f63b4c3360a106d82d8d75b1ce1dff731809099e Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 14:17:29 +0900 Subject: [PATCH 58/96] error fix --- CNN/edge_inference_request_scheduler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 4eae0e1..1f5d89f 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -105,9 +105,10 @@ def model_request(edge, model, order): threads = [] order = 0 for req in requests_list: - edge_to_inference = model_edge_info.get(req) - if edges_to_inference is not None: - edges_to_inference = edges_to_inference() + if req in model_edge_info.keys(): + edge_to_inference = model_edge_info.get(req)() + else: + edges_to_inference = None order += 1 th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) From f9cf1101202e180d380826c4f3a28ce52f4656f3 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 14:37:45 +0900 Subject: [PATCH 59/96] =?UTF-8?q?edges,=20models=20log=20=EC=B6=9C?= =?UTF-8?q?=EB=A0=A5=20=EC=BD=94=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 1f5d89f..cf89de9 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -41,6 +41,8 @@ print(f'--edge arg must be in {edges_register}') exit(1) +print(f'Edges to inference: {edges_to_inference}') + # 추론 요청 할 장비들에서 요청 가능한 모델들 models_to_inference = [] @@ -52,6 +54,8 @@ models_to_inference = set(models_to_inference) +print(f'Models to inference: {models_to_inference}') + # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 def model_request(edge, model, order): @@ -96,6 +100,8 @@ def model_request(edge, model, order): dataset = model_edge_info.get(model) model_edge_info[model] = roundrobin.smooth(dataset) +print(f'model-edge info: {model_edge_info}') + # 들어오는 요청들 임시 코드임!!! requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet', @@ -108,7 +114,7 @@ def model_request(edge, model, order): if req in model_edge_info.keys(): edge_to_inference = model_edge_info.get(req)() else: - edges_to_inference = None + edges_to_inference = '' order += 1 th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) From 92968613a9f1668a9bb0ecbcd4daf4773ddadc42 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 14:41:17 +0900 Subject: [PATCH 60/96] =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index cf89de9..54416c5 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -59,6 +59,9 @@ # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 def model_request(edge, model, order): + global edges_to_inference + global models_to_inference + if edge not in edges_to_inference: print(f'edge must be in {edges_to_inference}') return From 44f576abe50851d311af2d6b0eec8574b8708092 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 27 Feb 2023 15:11:18 +0900 Subject: [PATCH 61/96] error fix --- CNN/edge_inference_request_scheduler.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 54416c5..81e151b 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -59,15 +59,12 @@ # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 def model_request(edge, model, order): - global edges_to_inference - global models_to_inference - if edge not in edges_to_inference: - print(f'edge must be in {edges_to_inference}') + print(f'[{order}] edge must be in {edges_to_inference}/ input value: {edge}') return if model not in models_to_inference: - print(f'model must be in {models_to_inference}') + print(f'[{order}] model must be in {models_to_inference}/ input value: {model}') return req_processing_start_time = time.time() @@ -91,6 +88,9 @@ def model_request(edge, model, order): model_edge_info[model].append((edge, 1)) +print(f'model-edge dataset: {model_edge_info}') + + # for edge_info_key in edges_info.keys(): # edge_info = edges_info.get(edge_info_key) # for model in edge_info.get('model'): @@ -103,8 +103,6 @@ def model_request(edge, model, order): dataset = model_edge_info.get(model) model_edge_info[model] = roundrobin.smooth(dataset) -print(f'model-edge info: {model_edge_info}') - # 들어오는 요청들 임시 코드임!!! requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet', @@ -117,7 +115,7 @@ def model_request(edge, model, order): if req in model_edge_info.keys(): edge_to_inference = model_edge_info.get(req)() else: - edges_to_inference = '' + edge_to_inference = '' order += 1 th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) From 01baf731905ec1b6bc5598e0872f4971f26ff643 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:11:38 +0900 Subject: [PATCH 62/96] =?UTF-8?q?=EC=B6=94=EB=A1=A0=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=EC=8B=9C=20=EC=9A=94=EC=B2=AD=ED=95=A0=20=EB=AA=A8=EB=8D=B8?= =?UTF-8?q?=EB=93=A4=EC=9D=84=20=EB=AA=85=EB=A0=B9=ED=96=89=EC=9D=B8?= =?UTF-8?q?=EC=9E=90=EB=A1=9C=20=EB=B0=9B=EC=9D=84=20=EC=88=98=20=EC=9E=88?= =?UTF-8?q?=EB=8F=84=EB=A1=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 81e151b..24bd1b6 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -15,6 +15,13 @@ port = args.port +# 임시 코드 +parser.add_argument('--req', default='mobilenet,10', type=str) +inference_requests = args.req.spilt(',') +parser.add_argument('--random', action='store_true') +inference_random_flag = args.random + + # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] @@ -105,9 +112,17 @@ def model_request(edge, model, order): # 들어오는 요청들 임시 코드임!!! -requests_list = ['mobilenet', 'mobilenet_v2', 'mobilenet', 'mobilenet', - 'inception_v3', 'mobilenet_v2', 'inception_v3', 'mobilenet', - 'inception_v3', 'mobilenet_v2', 'inception_v3'] +requests_list = [] +for idx in range(0, len(inference_requests), 2): + model = inference_requests[idx] + inference_num = inference_requests[idx] + + for _ in range(inference_num): + requests_list.append(model) + +if inference_random_flag: + random.shuffle(requests_list) + threads = [] order = 0 From 710c43106cd4ae55e3f87b3a04bb8abce4ed03db Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:14:30 +0900 Subject: [PATCH 63/96] error fix --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 24bd1b6..8f981f3 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -16,8 +16,8 @@ # 임시 코드 -parser.add_argument('--req', default='mobilenet,10', type=str) -inference_requests = args.req.spilt(',') +parser.add_argument('--request', default='mobilenet,10', type=str) +inference_requests = args.request.split(',') parser.add_argument('--random', action='store_true') inference_random_flag = args.random From b4706214bf387c56f8245d9fb6498e9499a3bb29 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:16:08 +0900 Subject: [PATCH 64/96] error fix --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 8f981f3..1118b96 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -16,8 +16,8 @@ # 임시 코드 -parser.add_argument('--request', default='mobilenet,10', type=str) -inference_requests = args.request.split(',') +parser.add_argument('--reqs', default='mobilenet,10', type=str) +inference_requests = args.reqs.split(',') parser.add_argument('--random', action='store_true') inference_random_flag = args.random From 7472776dd9c1ad56e33e6a16e9b6b9332d55c829 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:21:32 +0900 Subject: [PATCH 65/96] error fix --- CNN/edge_inference_request_scheduler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 1118b96..d316e68 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -17,9 +17,12 @@ # 임시 코드 parser.add_argument('--reqs', default='mobilenet,10', type=str) -inference_requests = args.reqs.split(',') parser.add_argument('--random', action='store_true') -inference_random_flag = args.random + +temp_args = parser.parse_args() + +inference_requests = temp_args.reqs.split(',') +inference_random_flag = temp_args.random # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. From 95d5b0692d98d78f1ac615c4ca62979e84a94ca1 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:22:08 +0900 Subject: [PATCH 66/96] error fix --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index d316e68..ae4a0c8 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -118,7 +118,7 @@ def model_request(edge, model, order): requests_list = [] for idx in range(0, len(inference_requests), 2): model = inference_requests[idx] - inference_num = inference_requests[idx] + inference_num = int(inference_requests[idx]) for _ in range(inference_num): requests_list.append(model) From ab0305478f68899ffacd386606a0a821614487b1 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:23:13 +0900 Subject: [PATCH 67/96] error fix --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index ae4a0c8..27b74ca 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -118,7 +118,7 @@ def model_request(edge, model, order): requests_list = [] for idx in range(0, len(inference_requests), 2): model = inference_requests[idx] - inference_num = int(inference_requests[idx]) + inference_num = int(inference_requests[idx+1]) for _ in range(inference_num): requests_list.append(model) From 842450fc7ffe3358535a5362adf341d87cd1c055 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 28 Feb 2023 12:29:42 +0900 Subject: [PATCH 68/96] error fix --- CNN/edge_inference_request_scheduler.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 27b74ca..95ff4a7 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -9,13 +9,7 @@ parser.add_argument('--edge', default=None, type=str) parser.add_argument('--port', default=5001, type=int) -args = parser.parse_args() - -edges_to_inference = args.edge -port = args.port - - -# 임시 코드 +######### 임시 코드 ################### parser.add_argument('--reqs', default='mobilenet,10', type=str) parser.add_argument('--random', action='store_true') @@ -23,6 +17,15 @@ inference_requests = temp_args.reqs.split(',') inference_random_flag = temp_args.random +#################################### + +args = parser.parse_args() + +edges_to_inference = args.edge +port = args.port + + + # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. From 89b5679977bc180ef3e1306b684d19ce93b7e5f1 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 2 Mar 2023 13:12:38 +0900 Subject: [PATCH 69/96] sleep add --- CNN/edge_inference_request_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 95ff4a7..81b9052 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -142,6 +142,7 @@ def model_request(edge, model, order): th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) th.start() threads.append(th) + time.sleep(0.1) for th in threads: th.join() From 3e39b50b9ce6a1d2f783d33f576aa0da48921cbd Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 2 Mar 2023 18:22:13 +0900 Subject: [PATCH 70/96] =?UTF-8?q?=EB=B6=88=ED=95=84=EC=9A=94=ED=95=9C=20?= =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=A3=BC=EC=84=9D=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 81b9052..1492fc4 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -142,7 +142,7 @@ def model_request(edge, model, order): th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) th.start() threads.append(th) - time.sleep(0.1) + # time.sleep(0.1) for th in threads: th.join() From f7147bf14cb59ce604c774524d8ab382e5b612fc Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 3 Mar 2023 10:24:08 +0900 Subject: [PATCH 71/96] =?UTF-8?q?=EB=91=90=EA=B0=9C=EC=9D=98=20=EC=84=9C?= =?UTF-8?q?=EB=B2=84=EC=97=90=20=EC=9A=94=EC=B2=AD=EC=9D=84=20=EC=A0=84?= =?UTF-8?q?=EB=8B=AC=ED=95=A0=20=EC=88=98=20=EC=9E=88=EB=8A=94=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20=EC=BD=94=EB=93=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 1492fc4..ae19a3d 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -7,7 +7,6 @@ parser = argparse.ArgumentParser() parser.add_argument('--edge', default=None, type=str) -parser.add_argument('--port', default=5001, type=int) ######### 임시 코드 ################### parser.add_argument('--reqs', default='mobilenet,10', type=str) @@ -29,13 +28,13 @@ # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. -edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', +edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-tx2': {'url': f'http://192.168.0.8:{port}/', + 'nvidia-tx2': {'url': f'http://192.168.0.8:', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-nano1': {'url': f'http://192.168.0.29:{port}/', + 'nvidia-nano1': {'url': f'http://192.168.0.29:', 'model': ['mobilenet'] } } @@ -71,7 +70,7 @@ # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 -def model_request(edge, model, order): +def model_request(edge, model, port, order): if edge not in edges_to_inference: print(f'[{order}] edge must be in {edges_to_inference}/ input value: {edge}') return @@ -82,7 +81,7 @@ def model_request(edge, model, order): req_processing_start_time = time.time() edge_info = edges_info.get(edge) - url = edge_info.get('url') + model + url = edge_info.get('url') + f'{port}/' + model res = requests.get(url) processing_time = time.time() - req_processing_start_time print(f'[{order}] total request time: {processing_time}\n{res.text}') @@ -139,8 +138,11 @@ def model_request(edge, model, order): edge_to_inference = '' order += 1 - th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) + th = Thread(target=model_request, args=(edge_to_inference, req, 5001, f'{order}:{edge_to_inference}/{req}')) + th2 = Thread(target=model_request, args=(edge_to_inference, req, 5001, f'{order}:{edge_to_inference}/{req}')) th.start() + th2.start() + threads.append(th2) threads.append(th) # time.sleep(0.1) From 757bf64a14d8bcf2b3a4e377fb5834cb9ea829bd Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 3 Mar 2023 10:40:12 +0900 Subject: [PATCH 72/96] =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=EC=BD=94?= =?UTF-8?q?=EB=93=9C=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 28 +++++++------------------ 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index ae19a3d..d856234 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -7,6 +7,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--edge', default=None, type=str) +parser.add_argument('--port', default=5001, type=int) ######### 임시 코드 ################### parser.add_argument('--reqs', default='mobilenet,10', type=str) @@ -24,17 +25,14 @@ port = args.port - - - # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. -edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:', +edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-tx2': {'url': f'http://192.168.0.8:', + 'nvidia-tx2': {'url': f'http://192.168.0.8:{port}/', 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-nano1': {'url': f'http://192.168.0.29:', + 'nvidia-nano1': {'url': f'http://192.168.0.29:{port}/', 'model': ['mobilenet'] } } @@ -70,7 +68,7 @@ # 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 -def model_request(edge, model, port, order): +def model_request(edge, model, order): if edge not in edges_to_inference: print(f'[{order}] edge must be in {edges_to_inference}/ input value: {edge}') return @@ -81,7 +79,7 @@ def model_request(edge, model, port, order): req_processing_start_time = time.time() edge_info = edges_info.get(edge) - url = edge_info.get('url') + f'{port}/' + model + url = edge_info.get('url') + model res = requests.get(url) processing_time = time.time() - req_processing_start_time print(f'[{order}] total request time: {processing_time}\n{res.text}') @@ -103,14 +101,6 @@ def model_request(edge, model, port, order): print(f'model-edge dataset: {model_edge_info}') -# for edge_info_key in edges_info.keys(): -# edge_info = edges_info.get(edge_info_key) -# for model in edge_info.get('model'): -# if model not in model_edge_info.keys(): -# model_edge_info[model] = [] -# -# model_edge_info[model].append((edge_info_key, 1)) - for model in model_edge_info.keys(): dataset = model_edge_info.get(model) model_edge_info[model] = roundrobin.smooth(dataset) @@ -138,13 +128,9 @@ def model_request(edge, model, port, order): edge_to_inference = '' order += 1 - th = Thread(target=model_request, args=(edge_to_inference, req, 5001, f'{order}:{edge_to_inference}/{req}')) - th2 = Thread(target=model_request, args=(edge_to_inference, req, 5001, f'{order}:{edge_to_inference}/{req}')) + th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) th.start() - th2.start() - threads.append(th2) threads.append(th) - # time.sleep(0.1) for th in threads: th.join() From 0c42f6019ea26c60f3acdb7af3bb25b5e6dafd17 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 6 Mar 2023 20:36:25 +0900 Subject: [PATCH 73/96] =?UTF-8?q?=EC=97=A3=EC=A7=80=EC=9E=A5=EB=B9=84?= =?UTF-8?q?=EC=9D=98=20=EC=97=AC=EB=9F=AC=20=EC=84=9C=EB=B2=84=EB=A1=9C=20?= =?UTF-8?q?=EC=9A=94=EC=B2=AD=EC=9D=84=20=EC=A0=84=EB=8B=AC=20=ED=95=A0=20?= =?UTF-8?q?=EC=88=98=EC=9E=88=EB=8A=94=20=EA=B5=AC=EC=A1=B0=EB=A1=9C=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 112 ++++++++++++++++-------- 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index d856234..8b3cac3 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -7,7 +7,6 @@ parser = argparse.ArgumentParser() parser.add_argument('--edge', default=None, type=str) -parser.add_argument('--port', default=5001, type=int) ######### 임시 코드 ################### parser.add_argument('--reqs', default='mobilenet,10', type=str) @@ -22,18 +21,20 @@ args = parser.parse_args() edges_to_inference = args.edge -port = args.port # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. -edges_info = {'nvidia-xavier2': {'url': f'http://192.168.0.30:{port}/', - 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] +edges_info = {'nvidia-xavier2': {'ip_addr': '192.168.0.30', + 'ports': [5001, 5002], + 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-tx2': {'url': f'http://192.168.0.8:{port}/', - 'model': ['mobilenet', 'mobilenet_v2', 'inception_v3'] + 'nvidia-tx2': {'ip_addr': '192.168.0.8', + 'ports': [5001], + 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-nano1': {'url': f'http://192.168.0.29:{port}/', - 'model': ['mobilenet'] + 'nvidia-nano1': {'ip_addr': '192.168.0.29', + 'ports': [5001], + 'models': ['mobilenet'] } } @@ -59,7 +60,7 @@ for edge_name in edges_to_inference: edge_info = edges_info.get(edge_name) - models = edge_info.get('model') + models = edge_info.get('models') models_to_inference.extend(models) models_to_inference = set(models_to_inference) @@ -67,32 +68,13 @@ print(f'Models to inference: {models_to_inference}') -# 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 -def model_request(edge, model, order): - if edge not in edges_to_inference: - print(f'[{order}] edge must be in {edges_to_inference}/ input value: {edge}') - return - - if model not in models_to_inference: - print(f'[{order}] model must be in {models_to_inference}/ input value: {model}') - return - - req_processing_start_time = time.time() - edge_info = edges_info.get(edge) - url = edge_info.get('url') + model - res = requests.get(url) - processing_time = time.time() - req_processing_start_time - print(f'[{order}] total request time: {processing_time}\n{res.text}') - return - - # 현재 스케줄링 방식: 딕셔너리에 모델별로 엣지장비이름 등록, 들어오는 요청에 따라 각 장비들에 라운드로빈으로 스케줄링 # 문제점 각 모델이 하나씩 들어오면 장비들 중 하나에만 요청이 들어감 -> 고민 model_edge_info = {} for edge in edges_to_inference: edge_info = edges_info.get(edge) - for model in edge_info.get('model'): + for model in edge_info.get('models'): if model not in model_edge_info.keys(): model_edge_info[model] = [] @@ -100,13 +82,70 @@ def model_request(edge, model, order): print(f'model-edge dataset: {model_edge_info}') - for model in model_edge_info.keys(): dataset = model_edge_info.get(model) model_edge_info[model] = roundrobin.smooth(dataset) -# 들어오는 요청들 임시 코드임!!! +# 각 엣지 장비별 포트들을 라운드 로빈으로 얻어올 수 있도록 딕셔너리 구성 +edge_port_info = {} + +for edge in edges_to_inference: + if edge not in edge_port_info.keys(): + edge_port_info[edge] = [] + + edge_info = edges_info.get(edge) + for port in edge_info.get('ports'): + edge_port_info[edge].append((port, 1)) + +print(f'ip-edge dataset: {edge_port_info}') + +for edge in edge_port_info.keys(): + dataset = edge_port_info.get(edge) + edge_port_info[edge] = roundrobin.smooth(dataset) + + +# 모델로 엣지장비 이름 얻는 함수, 모델마다 엣지장비들의 정보가 담겨 있고 얻어올 때는 라운드로빈으로 순서대로 가져옴 +def get_edge_by_model_rr(model): + if model in model_edge_info.keys(): + return model_edge_info.get(model)() + else: + return None + + +# 엣지장비 이름으로 해당 포트번호를 얻는 함수, 엣지장비마다 포트 번호 정보가 담겨 있고 라운드로빈으로 순서대로 가져옴 +def get_port_by_edge_rr(edge): + if edge in edge_port_info.keys(): + return edge_port_info.get(edge)() + else: + return None + + +# 추론을 요청하는 함수, 인자로는 추론을 요청할 엣지 장비, 모델, 요청임. 엣지장비와 모델은 위의 edges_info에 등록되어 있어야함 +def model_request(edge, model, order): + if edge not in edges_to_inference: + print(f'[{order}] edge must be in {edges_to_inference}/ input value: {edge}') + return + + if model not in models_to_inference: + print(f'[{order}] model must be in {models_to_inference}/ input value: {model}') + return + + edge_info = edges_info.get(edge) + + edge_ip_addr = edge_info.get('ip_addr') + port = get_port_by_edge_rr(edge) + url = f'http://{edge_ip_addr}:{port}/{model}' + + req_processing_start_time = time.time() + res = requests.get(url) + processing_time = time.time() - req_processing_start_time + + print(f'[{order}] total request time: {processing_time}\n{res.text}') + return + + +### 들어오는 요청들 임시 코드임!!! ### requests_list = [] for idx in range(0, len(inference_requests), 2): model = inference_requests[idx] @@ -117,15 +156,18 @@ def model_request(edge, model, order): if inference_random_flag: random.shuffle(requests_list) +############################## +# 요청을 각 장비에 전달, 여러요청을 동시에 다룰 수 있도록 쓰레드 이용 threads = [] order = 0 + for req in requests_list: - if req in model_edge_info.keys(): - edge_to_inference = model_edge_info.get(req)() - else: - edge_to_inference = '' + edge_to_inference = get_edge_by_model_rr(req) + if edge_to_inference is None: + print(f'{req} can\'t be inference') + continue order += 1 th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) From b509c37b302f9a5e19101c805557a40d5b26b19b Mon Sep 17 00:00:00 2001 From: kh3654po Date: Mon, 6 Mar 2023 20:49:31 +0900 Subject: [PATCH 74/96] =?UTF-8?q?=EA=B2=B0=EA=B3=BC=EA=B0=92=EC=9D=84=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=ED=95=A0=20=EB=95=8C=20=EA=B2=B0=EA=B3=BC?= =?UTF-8?q?=EA=B0=80=20=EC=98=A8=20=EC=84=9C=EB=B2=84=EC=9D=98=20=ED=8F=AC?= =?UTF-8?q?=ED=8A=B8=EB=B2=88=ED=98=B8=EB=8F=84=20=EC=B6=9C=EB=A0=A5?= =?UTF-8?q?=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 8b3cac3..269cb3a 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -141,7 +141,7 @@ def model_request(edge, model, order): res = requests.get(url) processing_time = time.time() - req_processing_start_time - print(f'[{order}] total request time: {processing_time}\n{res.text}') + print(f'[{order}:{edge}({port})/{model}] total request time: {processing_time}\n{res.text}') return @@ -170,7 +170,7 @@ def model_request(edge, model, order): continue order += 1 - th = Thread(target=model_request, args=(edge_to_inference, req, f'{order}:{edge_to_inference}/{req}')) + th = Thread(target=model_request, args=(edge_to_inference, req, order)) th.start() threads.append(th) From 7d686586ed9810676074d43f7b3a84fe2376a32d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 7 Mar 2023 11:47:51 +0900 Subject: [PATCH 75/96] =?UTF-8?q?=EC=B6=94=EB=A1=A0=EC=9D=B4=20=EC=99=84?= =?UTF-8?q?=EB=A3=8C=EB=90=98=EB=A9=B4=20=EA=B2=B0=EA=B3=BC=EB=A5=BC=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=ED=95=98=EB=8F=84=EB=A1=9D=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 269cb3a..25fd4d9 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -141,6 +141,10 @@ def model_request(edge, model, order): res = requests.get(url) processing_time = time.time() - req_processing_start_time + inference_time = res.text.split(':')[1] + inference_time = inference_time.split('\n')[0] + inference_time_results[order-1] = float(inference_time) + print(f'[{order}:{edge}({port})/{model}] total request time: {processing_time}\n{res.text}') return @@ -163,6 +167,8 @@ def model_request(edge, model, order): threads = [] order = 0 +inference_time_results = [0 for _ in range(len(requests_list))] + for req in requests_list: edge_to_inference = get_edge_by_model_rr(req) if edge_to_inference is None: @@ -176,3 +182,22 @@ def model_request(edge, model, order): for th in threads: th.join() + + +# 추론요청 결과 출력 (최소, 최대, 총, 평균) +inference_time_results.sort() +len_inference_time_results = len(inference_time_results) + +total_inference_time = sum(inference_time_results) +avg_inference_time = total_inference_time / len_inference_time_results +min_inference_time = inference_time_results[0] +mid_inference_time = inference_time_results[int(len_inference_time_results / 2)] +max_inference_time = inference_time_results[-1] + +print(f'\n총 추론 시간: {total_inference_time}') +print(f'평균 추론 시간: {avg_inference_time}') +print(f'최소 추론 시간: {min_inference_time}') +print(f'중간 추론 시간: {mid_inference_time}') +print(f'최대 추론 시간: {max_inference_time}\n') + + From a55fe58d087daeba37e83d6eafe9c417932499a5 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 7 Mar 2023 11:51:17 +0900 Subject: [PATCH 76/96] =?UTF-8?q?=EC=B6=9C=EB=A0=A5=20=EA=B2=B0=EA=B3=BC?= =?UTF-8?q?=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 25fd4d9..7e53ccd 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -184,7 +184,7 @@ def model_request(edge, model, order): th.join() -# 추론요청 결과 출력 (최소, 최대, 총, 평균) +# 추론요청 결과 출력 (최소, 중간, 최대, 평균) inference_time_results.sort() len_inference_time_results = len(inference_time_results) @@ -194,7 +194,6 @@ def model_request(edge, model, order): mid_inference_time = inference_time_results[int(len_inference_time_results / 2)] max_inference_time = inference_time_results[-1] -print(f'\n총 추론 시간: {total_inference_time}') print(f'평균 추론 시간: {avg_inference_time}') print(f'최소 추론 시간: {min_inference_time}') print(f'중간 추론 시간: {mid_inference_time}') From fc66abb943488a6c79ba7cb4331193ece4dc92c8 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 7 Mar 2023 17:10:38 +0900 Subject: [PATCH 77/96] =?UTF-8?q?=EA=B0=81=20=EC=9E=A5=EB=B9=84=EC=9D=98?= =?UTF-8?q?=20ip=20=EC=A0=95=EB=B3=B4=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 7e53ccd..ef307ca 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -24,15 +24,15 @@ # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. -edges_info = {'nvidia-xavier2': {'ip_addr': '192.168.0.30', +edges_info = {'nvidia-xavier2': {'ip_addr': '192.168.0.32', 'ports': [5001, 5002], 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-tx2': {'ip_addr': '192.168.0.8', + 'nvidia-tx2': {'ip_addr': '192.168.0.22', 'ports': [5001], 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] }, - 'nvidia-nano1': {'ip_addr': '192.168.0.29', + 'nvidia-nano1': {'ip_addr': '192.168.0.41', 'ports': [5001], 'models': ['mobilenet'] } From 4f5621ab6870118bfabe4cd591f41812b4d10ebd Mon Sep 17 00:00:00 2001 From: kh3654po Date: Wed, 8 Mar 2023 15:38:26 +0900 Subject: [PATCH 78/96] =?UTF-8?q?=EC=84=9C=EB=B2=84=20=EB=8F=99=EC=9E=91?= =?UTF-8?q?=EC=8B=9C=20=EC=8B=B1=EA=B8=80=20=EC=93=B0=EB=A0=88=EB=93=9C?= =?UTF-8?q?=EB=A1=9C=20=EB=8F=99=EC=9E=91=ED=95=98=EB=8F=84=EB=A1=9D=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 8f3215c..7cda012 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -107,19 +107,12 @@ def save_model(model, saved_model_dir): app = Flask(__name__) -# order = 0 - @app.route('/mobilenet') def mobilenetv1(): - # global order - # order += 1 - # my_order = order inference_start_time = time.time() - # print(f'[{my_order}] inference start: {inference_start_time}') result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) inference_end_time = time.time() - # print(f'[{my_order}] inference end: {inference_end_time}') inference_time = inference_end_time - inference_start_time @@ -152,4 +145,4 @@ def inceptionv3(): return f'inceptionv3 inference success\ninference time:{inference_time}\n' -app.run(host=hostname, port=port) +app.run(host=hostname, port=port, threaded=False) From ae18e851c5768c7091c7caca1d91874c5f91050d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Wed, 8 Mar 2023 15:39:11 +0900 Subject: [PATCH 79/96] =?UTF-8?q?=EC=9A=94=EC=B2=AD=EC=9D=84=20=EB=B3=B4?= =?UTF-8?q?=EB=82=BC=EB=95=8C=201=EC=B4=88=EC=97=90=20=EA=B1=B8=EC=B3=90?= =?UTF-8?q?=20=EB=B3=B4=EB=82=B4=EB=8F=84=EB=A1=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index ef307ca..e1af75f 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -98,7 +98,7 @@ for port in edge_info.get('ports'): edge_port_info[edge].append((port, 1)) -print(f'ip-edge dataset: {edge_port_info}') +print(f'edge-port dataset: {edge_port_info}') for edge in edge_port_info.keys(): dataset = edge_port_info.get(edge) @@ -168,6 +168,7 @@ def model_request(edge, model, order): order = 0 inference_time_results = [0 for _ in range(len(requests_list))] +request_sleep_time = 1 / len(requests_list) # 요청들을 1초에 나눠서 보내기 위한 슬립시간 for req in requests_list: edge_to_inference = get_edge_by_model_rr(req) @@ -179,6 +180,7 @@ def model_request(edge, model, order): th = Thread(target=model_request, args=(edge_to_inference, req, order)) th.start() threads.append(th) + time.sleep(request_sleep_time) for th in threads: th.join() From f295a68938959ceb0f1294dcb130c93887270752 Mon Sep 17 00:00:00 2001 From: Jungae Park <46885199+jungae-park@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:25:40 +0900 Subject: [PATCH 80/96] Update object_detect_raw_image_classification_inference.py code import path change --- ...object_detect_raw_image_classification_inference.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CNN/object_detect_raw_image_classification_inference.py b/CNN/object_detect_raw_image_classification_inference.py index e69a1e5..88b128b 100644 --- a/CNN/object_detect_raw_image_classification_inference.py +++ b/CNN/object_detect_raw_image_classification_inference.py @@ -12,12 +12,12 @@ sys.path.append(str(ROOT)) # add ROOT to PATH ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative -from models.common import DetectMultiBackend -from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams -from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, +from model.yolov5.models.common import DetectMultiBackend +from model.yolov5.utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from model.yolov5.utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) -from utils.plots import Annotator, colors, save_one_box -from utils.torch_utils import select_device, smart_inference_mode +from model.yolov5.utils.plots import Annotator, colors, save_one_box +from model.yolov5.utils.torch_utils import select_device, smart_inference_mode @smart_inference_mode() From 0a8a78cac497227023eb27aee1e8e7626d7cb2bf Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 12 Mar 2023 22:19:04 +0900 Subject: [PATCH 81/96] =?UTF-8?q?yolo=20v5=20=ED=85=8C=EC=8A=A4=ED=8A=B8?= =?UTF-8?q?=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 323 ++++++++++++++++----------- 1 file changed, 191 insertions(+), 132 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 7cda012..44bed5e 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -1,5 +1,12 @@ import argparse +from model.yolov5.models.common import DetectMultiBackend +from model.yolov5.utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from model.yolov5.utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +import torch +from pathlib import Path + from flask import Flask import tensorflow as tf import numpy as np @@ -14,135 +21,187 @@ [tf.config.experimental.VirtualDeviceConfiguration( memory_limit=0.6 * 1024)]) -from tensorflow.keras.applications import ( - mobilenet, - mobilenet_v2, - inception_v3 -) - -parser = argparse.ArgumentParser() -parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) -parser.add_argument('--hostname', default='0.0.0.0', type=str) -parser.add_argument('--port', default=5001, type=int) -args = parser.parse_args() -models_to_load = args.model.split(',') -hostname = args.hostname -port = args.port - -models = { - 'mobilenet': mobilenet, - 'mobilenet_v2': mobilenet_v2, - 'inception_v3': inception_v3 -} - -models_detail = { - 'mobilenet': mobilenet.MobileNet(weights='imagenet'), - 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), - 'inception_v3': inception_v3.InceptionV3(weights='imagenet') -} - - -def mobilenet_load_image(image_path): - return tf.keras.preprocessing.image.load_img( - image_path, - target_size=[224, 224]) - - -def inception_load_image(image_path): - return tf.keras.preprocessing.image.load_img( - image_path, - target_size=[299, 299]) - - -def image_to_array(image): - return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) - - -def image_preprocess(image_array, model_name): - return models[model_name].preprocess_input( - image_array[tf.newaxis, ...]) - -print('\npreprossing images...') - -mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' -mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' -inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' - -mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) -mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) -inceptionv3_test_image = inception_load_image(inceptionv3_image_path) - -mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) -mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) -inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) - -mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') -mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') -inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') - -print('image preprocessing completed!\n') - -def save_model(model, saved_model_dir): - model = models_detail[model] - shutil.rmtree(saved_model_dir, ignore_errors=True) - model.save(saved_model_dir, include_optimizer=False, save_format='tf') - -print('\nsaving and loading models...') - -loaded_models = {} - -for model_name in models_to_load: - model_names = models_detail.keys() - if model_name in model_names: - model_path = f'{model_name}_saved_model' - if os.path.isdir(model_path) == False: - print('model save') - save_model(model_name, model_path) - loaded_models[model_name] = tf.keras.models.load_model(model_path) - else: - print(f'model names must be in {model_names}') - exit(1) - -print('saving and loading models completed!\n') - -app = Flask(__name__) - - -@app.route('/mobilenet') -def mobilenetv1(): - inference_start_time = time.time() - result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) - inference_end_time = time.time() - - inference_time = inference_end_time - inference_start_time - - return f'mobilenetv1 inference success\ninference time:{inference_time}\n' - - -@app.route('/mobilenet_v2') -def mobilenetv2(): - inference_start_time = time.time() - result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) - inference_end_time = time.time() - - inference_time = inference_end_time - inference_start_time - - # print(result) - - return f'mobilenetv2 inference success\ninference time:{inference_time}\n' - - -@app.route('/inception_v3') -def inceptionv3(): - inference_start_time = time.time() - result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) - inference_end_time = time.time() - - inference_time = inference_end_time - inference_start_time - - # print(result) - - return f'inceptionv3 inference success\ninference time:{inference_time}\n' - - -app.run(host=hostname, port=port, threaded=False) +# from tensorflow.keras.applications import ( +# mobilenet, +# mobilenet_v2, +# inception_v3 +# ) +# +# parser = argparse.ArgumentParser() +# parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +# parser.add_argument('--hostname', default='0.0.0.0', type=str) +# parser.add_argument('--port', default=5001, type=int) +# args = parser.parse_args() +# models_to_load = args.model.split(',') +# hostname = args.hostname +# port = args.port +# +# models = { +# 'mobilenet': mobilenet, +# 'mobilenet_v2': mobilenet_v2, +# 'inception_v3': inception_v3 +# } +# +# models_detail = { +# 'mobilenet': mobilenet.MobileNet(weights='imagenet'), +# 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), +# 'inception_v3': inception_v3.InceptionV3(weights='imagenet') +# } +# +# +# def mobilenet_load_image(image_path): +# return tf.keras.preprocessing.image.load_img( +# image_path, +# target_size=[224, 224]) +# +# +# def inception_load_image(image_path): +# return tf.keras.preprocessing.image.load_img( +# image_path, +# target_size=[299, 299]) +# +# +# def image_to_array(image): +# return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) +# +# +# def image_preprocess(image_array, model_name): +# return models[model_name].preprocess_input( +# image_array[tf.newaxis, ...]) +# +# print('\npreprossing images...') +# +# mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' +# mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' +# inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' +# +# mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) +# mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) +# inceptionv3_test_image = inception_load_image(inceptionv3_image_path) +# +# mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) +# mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) +# inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) +# +# mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') +# mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') +# inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') +# +# print('image preprocessing completed!\n') +# +# def save_model(model, saved_model_dir): +# model = models_detail[model] +# shutil.rmtree(saved_model_dir, ignore_errors=True) +# model.save(saved_model_dir, include_optimizer=False, save_format='tf') +# +# print('\nsaving and loading models...') +# +# loaded_models = {} +# +# for model_name in models_to_load: +# model_names = models_detail.keys() +# if model_name in model_names: +# model_path = f'{model_name}_saved_model' +# if os.path.isdir(model_path) == False: +# print('model save') +# save_model(model_name, model_path) +# loaded_models[model_name] = tf.keras.models.load_model(model_path) +# else: +# print(f'model names must be in {model_names}') +# exit(1) +# +# print('saving and loading models completed!\n') + + +# Yolo v5 +yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' +weights = './model/yolov5/yolov5s.pt' +data = './model/yolov5/coco128.yaml' # dataset.yaml path +imgsz = (640, 640) # inference size (height, width) +conf_thres = 0.25 # confidence threshold +iou_thres = 0.45 # NMS IOU threshold +max_det = 1000 # maximum detections per image +vid_stride = 1, # video frame-rate stride + +yolo_model = DetectMultiBackend(weights, data=data) +stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt +imgsz = check_img_size(imgsz, s=stride) # check image size + +# Dataloader +bs = 1 # batch_size +dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + +# Run inference +yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup +seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) +for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(yolo_model.device) + im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + pred = yolo_model(im) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + p = Path(p) # to Path + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + +# Print results +t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image +LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + +# +# app = Flask(__name__) +# +# +# @app.route('/mobilenet') +# def mobilenetv1(): +# inference_start_time = time.time() +# result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) +# inference_end_time = time.time() +# +# inference_time = inference_end_time - inference_start_time +# +# return f'mobilenetv1 inference success\ninference time:{inference_time}\n' +# +# +# @app.route('/mobilenet_v2') +# def mobilenetv2(): +# inference_start_time = time.time() +# result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) +# inference_end_time = time.time() +# +# inference_time = inference_end_time - inference_start_time +# +# # print(result) +# +# return f'mobilenetv2 inference success\ninference time:{inference_time}\n' +# +# +# @app.route('/inception_v3') +# def inceptionv3(): +# inference_start_time = time.time() +# result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) +# inference_end_time = time.time() +# +# inference_time = inference_end_time - inference_start_time +# +# # print(result) +# +# return f'inceptionv3 inference success\ninference time:{inference_time}\n' +# +# +# app.run(host=hostname, port=port, threaded=False) From ab97901ad2809a986e6793467d2204f764e854bc Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 12 Mar 2023 22:29:41 +0900 Subject: [PATCH 82/96] =?UTF-8?q?yolo=20v5=20falsk=20=EC=84=9C=EB=B2=84?= =?UTF-8?q?=EC=97=90=20=EC=98=AC=EB=A6=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 248 ++++++++++++++------------- 1 file changed, 131 insertions(+), 117 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 44bed5e..d9bb3e4 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -21,82 +21,82 @@ [tf.config.experimental.VirtualDeviceConfiguration( memory_limit=0.6 * 1024)]) -# from tensorflow.keras.applications import ( -# mobilenet, -# mobilenet_v2, -# inception_v3 -# ) -# -# parser = argparse.ArgumentParser() -# parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) -# parser.add_argument('--hostname', default='0.0.0.0', type=str) -# parser.add_argument('--port', default=5001, type=int) -# args = parser.parse_args() -# models_to_load = args.model.split(',') -# hostname = args.hostname -# port = args.port -# -# models = { -# 'mobilenet': mobilenet, -# 'mobilenet_v2': mobilenet_v2, -# 'inception_v3': inception_v3 -# } -# -# models_detail = { -# 'mobilenet': mobilenet.MobileNet(weights='imagenet'), -# 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), -# 'inception_v3': inception_v3.InceptionV3(weights='imagenet') -# } -# -# -# def mobilenet_load_image(image_path): -# return tf.keras.preprocessing.image.load_img( -# image_path, -# target_size=[224, 224]) -# -# -# def inception_load_image(image_path): -# return tf.keras.preprocessing.image.load_img( -# image_path, -# target_size=[299, 299]) -# -# -# def image_to_array(image): -# return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) -# -# -# def image_preprocess(image_array, model_name): -# return models[model_name].preprocess_input( -# image_array[tf.newaxis, ...]) -# -# print('\npreprossing images...') -# -# mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' -# mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' -# inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' -# -# mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) -# mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) -# inceptionv3_test_image = inception_load_image(inceptionv3_image_path) -# -# mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) -# mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) -# inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) -# -# mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') -# mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') -# inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') -# -# print('image preprocessing completed!\n') -# -# def save_model(model, saved_model_dir): -# model = models_detail[model] -# shutil.rmtree(saved_model_dir, ignore_errors=True) -# model.save(saved_model_dir, include_optimizer=False, save_format='tf') +from tensorflow.keras.applications import ( + mobilenet, + mobilenet_v2, + inception_v3 +) + +parser = argparse.ArgumentParser() +parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +parser.add_argument('--hostname', default='0.0.0.0', type=str) +parser.add_argument('--port', default=5001, type=int) +args = parser.parse_args() +models_to_load = args.model.split(',') +hostname = args.hostname +port = args.port + +models = { + 'mobilenet': mobilenet, + 'mobilenet_v2': mobilenet_v2, + 'inception_v3': inception_v3 +} + +models_detail = { + 'mobilenet': mobilenet.MobileNet(weights='imagenet'), + 'mobilenet_v2': mobilenet_v2.MobileNetV2(weights='imagenet'), + 'inception_v3': inception_v3.InceptionV3(weights='imagenet') +} + + +def mobilenet_load_image(image_path): + return tf.keras.preprocessing.image.load_img( + image_path, + target_size=[224, 224]) + + +def inception_load_image(image_path): + return tf.keras.preprocessing.image.load_img( + image_path, + target_size=[299, 299]) + + +def image_to_array(image): + return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) + + +def image_preprocess(image_array, model_name): + return models[model_name].preprocess_input( + image_array[tf.newaxis, ...]) + +print('\npreprossing images...') + +mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' +mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' +inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' + +mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) +mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) +inceptionv3_test_image = inception_load_image(inceptionv3_image_path) + +mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) +mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) +inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) + +mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') +mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') +inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') + +print('image preprocessing completed!\n') + +def save_model(model, saved_model_dir): + model = models_detail[model] + shutil.rmtree(saved_model_dir, ignore_errors=True) + model.save(saved_model_dir, include_optimizer=False, save_format='tf') # # print('\nsaving and loading models...') # -# loaded_models = {} +loaded_models = {} # # for model_name in models_to_load: # model_names = models_detail.keys() @@ -129,12 +129,12 @@ # Dataloader bs = 1 # batch_size -dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) +yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) # Run inference yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) -for path, im, im0s, vid_cap, s in dataset: +for path, im, im0s, vid_cap, s in yolov5_dataset: with dt[0]: im = torch.from_numpy(im).to(yolo_model.device) im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 @@ -159,49 +159,63 @@ # Print time (inference-only) LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + # Print results t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) -# -# app = Flask(__name__) -# -# -# @app.route('/mobilenet') -# def mobilenetv1(): -# inference_start_time = time.time() -# result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) -# inference_end_time = time.time() -# -# inference_time = inference_end_time - inference_start_time -# -# return f'mobilenetv1 inference success\ninference time:{inference_time}\n' -# -# -# @app.route('/mobilenet_v2') -# def mobilenetv2(): -# inference_start_time = time.time() -# result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) -# inference_end_time = time.time() -# -# inference_time = inference_end_time - inference_start_time -# -# # print(result) -# -# return f'mobilenetv2 inference success\ninference time:{inference_time}\n' -# -# -# @app.route('/inception_v3') -# def inceptionv3(): -# inference_start_time = time.time() -# result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) -# inference_end_time = time.time() -# -# inference_time = inference_end_time - inference_start_time -# -# # print(result) -# -# return f'inceptionv3 inference success\ninference time:{inference_time}\n' -# -# -# app.run(host=hostname, port=port, threaded=False) + +app = Flask(__name__) + + +@app.route('/mobilenet') +def mobilenetv1(): + inference_start_time = time.time() + result = loaded_models['mobilenet'].predict(mobilenetv1_test_image_preprocessed) + inference_end_time = time.time() + + inference_time = inference_end_time - inference_start_time + + return f'mobilenetv1 inference success\ninference time:{inference_time}\n' + + +@app.route('/mobilenet_v2') +def mobilenetv2(): + inference_start_time = time.time() + result = loaded_models['mobilenet_v2'].predict(mobilenetv2_test_image_preprocessed) + inference_end_time = time.time() + + inference_time = inference_end_time - inference_start_time + + # print(result) + + return f'mobilenetv2 inference success\ninference time:{inference_time}\n' + + +@app.route('/inception_v3') +def inceptionv3(): + inference_start_time = time.time() + result = loaded_models['inception_v3'].predict(inceptionv3_test_image_preprocessed) + inference_end_time = time.time() + + inference_time = inference_end_time - inference_start_time + + # print(result) + + return f'inceptionv3 inference success\ninference time:{inference_time}\n' + + +@app.route('/yolo_v5') +def yolov5(): + path, im, im0s, vid_cap, s = yolov5_dataset + + inference_start_time = time.time() + result = yolo_model(im) + inference_end_time = time.time() + + inference_time = inference_end_time - inference_start_time + + return f'inceptionv3 inference success\ninference time:{inference_time}\n' + + +app.run(host=hostname, port=port, threaded=False) From c540dd65c41f592932bd2fa69cea811b9554cbf8 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 12 Mar 2023 22:31:44 +0900 Subject: [PATCH 83/96] error fix --- CNN/edge_inference_request_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index d9bb3e4..ff77852 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -153,7 +153,7 @@ def save_model(model, saved_model_dir): # Process predictions for i, det in enumerate(pred): # per image seen += 1 - p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) p = Path(p) # to Path # Print time (inference-only) @@ -214,7 +214,7 @@ def yolov5(): inference_end_time = time.time() inference_time = inference_end_time - inference_start_time - + return f'inceptionv3 inference success\ninference time:{inference_time}\n' From 83677307682e7f494d3b1f6525b4999a29083de6 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Sun, 12 Mar 2023 22:39:33 +0900 Subject: [PATCH 84/96] error fix --- CNN/edge_inference_request_server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index ff77852..e27b263 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -207,13 +207,14 @@ def inceptionv3(): @app.route('/yolo_v5') def yolov5(): - path, im, im0s, vid_cap, s = yolov5_dataset + inference_time = 0 + for path, im, im0s, vid_cap, s in yolov5_dataset: - inference_start_time = time.time() - result = yolo_model(im) - inference_end_time = time.time() + inference_start_time = time.time() + result = yolo_model(im) + inference_end_time = time.time() - inference_time = inference_end_time - inference_start_time + inference_time = inference_end_time - inference_start_time return f'inceptionv3 inference success\ninference time:{inference_time}\n' From 232b7ceddb50b6da6f579758927fa016d5f8a8be Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 17:13:09 +0900 Subject: [PATCH 85/96] yolo v5 test --- CNN/edge_inference_request_server.py | 67 ++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index e27b263..d8d41c2 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -207,16 +207,67 @@ def inceptionv3(): @app.route('/yolo_v5') def yolov5(): - inference_time = 0 - for path, im, im0s, vid_cap, s in yolov5_dataset: - - inference_start_time = time.time() - result = yolo_model(im) - inference_end_time = time.time() + # for path, im, im0s, vid_cap, s in yolov5_dataset: + # + # inference_start_time = time.time() + # result = yolo_model(im) + # inference_end_time = time.time() + # + # inference_time = inference_end_time - inference_start_time - inference_time = inference_end_time - inference_start_time + inference_start_time = time.time() + yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' + weights = './model/yolov5/yolov5s.pt' + data = './model/yolov5/coco128.yaml' # dataset.yaml path + imgsz = (640, 640) # inference size (height, width) + conf_thres = 0.25 # confidence threshold + iou_thres = 0.45 # NMS IOU threshold + max_det = 1000 # maximum detections per image + vid_stride = 1, # video frame-rate stride + + yolo_model = DetectMultiBackend(weights, data=data) + stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + + # Run inference + yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + for path, im, im0s, vid_cap, s in yolov5_dataset: + with dt[0]: + im = torch.from_numpy(im).to(yolo_model.device) + im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + pred = yolo_model(im) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) + p = Path(p) # to Path + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + inference_end_time = time.time() - return f'inceptionv3 inference success\ninference time:{inference_time}\n' + inference_time = inference_end_time - inference_start_time + return f'yolov5 inference success\ninference time:{inference_time}\n' app.run(host=hostname, port=port, threaded=False) From b314e3fde788fc2be5e88e4ea3e4d949601256b7 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 17:51:07 +0900 Subject: [PATCH 86/96] yolo v5 test --- CNN/edge_inference_request_server.py | 181 ++++++++++++++------------- 1 file changed, 95 insertions(+), 86 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index d8d41c2..d6ca333 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -2,8 +2,10 @@ from model.yolov5.models.common import DetectMultiBackend from model.yolov5.utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams -from model.yolov5.utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, - increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from model.yolov5.utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, + colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, + xyxy2xywh) import torch from pathlib import Path @@ -69,6 +71,7 @@ def image_preprocess(image_array, model_name): return models[model_name].preprocess_input( image_array[tf.newaxis, ...]) + print('\npreprossing images...') mobilenetv1_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' @@ -89,10 +92,13 @@ def image_preprocess(image_array, model_name): print('image preprocessing completed!\n') + def save_model(model, saved_model_dir): model = models_detail[model] shutil.rmtree(saved_model_dir, ignore_errors=True) model.save(saved_model_dir, include_optimizer=False, save_format='tf') + + # # print('\nsaving and loading models...') # @@ -134,35 +140,37 @@ def save_model(model, saved_model_dir): # Run inference yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) -for path, im, im0s, vid_cap, s in yolov5_dataset: - with dt[0]: - im = torch.from_numpy(im).to(yolo_model.device) - im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 - im /= 255 # 0 - 255 to 0.0 - 1.0 - if len(im.shape) == 3: - im = im[None] # expand for batch dim - - # Inference - with dt[1]: - pred = yolo_model(im) - - # NMS - with dt[2]: - pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) - - # Process predictions - for i, det in enumerate(pred): # per image - seen += 1 - p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) - p = Path(p) # to Path - - # Print time (inference-only) - LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") - - -# Print results -t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image -LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) +path, im, im0s, vid_cap, s = yolov5_dataset[0] + +# for path, im, im0s, vid_cap, s in yolov5_dataset: +# with dt[0]: +# im = torch.from_numpy(im).to(yolo_model.device) +# im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 +# im /= 255 # 0 - 255 to 0.0 - 1.0 +# if len(im.shape) == 3: +# im = im[None] # expand for batch dim +# +# # Inference +# with dt[1]: +# pred = yolo_model(im) +# +# # NMS +# with dt[2]: +# pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) +# +# # Process predictions +# for i, det in enumerate(pred): # per image +# seen += 1 +# p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) +# p = Path(p) # to Path +# +# # Print time (inference-only) +# LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") +# +# +# # Print results +# t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image +# LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) app = Flask(__name__) @@ -209,65 +217,66 @@ def inceptionv3(): def yolov5(): # for path, im, im0s, vid_cap, s in yolov5_dataset: # - # inference_start_time = time.time() - # result = yolo_model(im) - # inference_end_time = time.time() - # - # inference_time = inference_end_time - inference_start_time - inference_start_time = time.time() - yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' - weights = './model/yolov5/yolov5s.pt' - data = './model/yolov5/coco128.yaml' # dataset.yaml path - imgsz = (640, 640) # inference size (height, width) - conf_thres = 0.25 # confidence threshold - iou_thres = 0.45 # NMS IOU threshold - max_det = 1000 # maximum detections per image - vid_stride = 1, # video frame-rate stride - - yolo_model = DetectMultiBackend(weights, data=data) - stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt - imgsz = check_img_size(imgsz, s=stride) # check image size - - # Dataloader - bs = 1 # batch_size - yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) - - # Run inference - yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup - seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) - for path, im, im0s, vid_cap, s in yolov5_dataset: - with dt[0]: - im = torch.from_numpy(im).to(yolo_model.device) - im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 - im /= 255 # 0 - 255 to 0.0 - 1.0 - if len(im.shape) == 3: - im = im[None] # expand for batch dim - - # Inference - with dt[1]: - pred = yolo_model(im) - - # NMS - with dt[2]: - pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) - - # Process predictions - for i, det in enumerate(pred): # per image - seen += 1 - p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) - p = Path(p) # to Path - - # Print time (inference-only) - LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") - - # Print results - t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image - LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + result = yolo_model(im) inference_end_time = time.time() inference_time = inference_end_time - inference_start_time - return f'yolov5 inference success\ninference time:{inference_time}\n' +# +# inference_start_time = time.time() +# yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' +# weights = './model/yolov5/yolov5s.pt' +# data = './model/yolov5/coco128.yaml' # dataset.yaml path +# imgsz = (640, 640) # inference size (height, width) +# conf_thres = 0.25 # confidence threshold +# iou_thres = 0.45 # NMS IOU threshold +# max_det = 1000 # maximum detections per image +# vid_stride = 1, # video frame-rate stride +# +# yolo_model = DetectMultiBackend(weights, data=data) +# stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt +# imgsz = check_img_size(imgsz, s=stride) # check image size +# +# # Dataloader +# bs = 1 # batch_size +# yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) +# +# # Run inference +# yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup +# seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) +# for path, im, im0s, vid_cap, s in yolov5_dataset: +# with dt[0]: +# im = torch.from_numpy(im).to(yolo_model.device) +# im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 +# im /= 255 # 0 - 255 to 0.0 - 1.0 +# if len(im.shape) == 3: +# im = im[None] # expand for batch dim +# +# # Inference +# with dt[1]: +# pred = yolo_model(im) +# +# # NMS +# with dt[2]: +# pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) +# +# # Process predictions +# for i, det in enumerate(pred): # per image +# seen += 1 +# p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) +# p = Path(p) # to Path +# +# # Print time (inference-only) +# LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") +# +# # Print results +# t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image +# LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) +# inference_end_time = time.time() +# +# inference_time = inference_end_time - inference_start_time + return f'yolov5 inference success\ninference time:{inference_time}\n' + app.run(host=hostname, port=port, threaded=False) From bb14eebc61c3f3cfb515c476c00b950c7776399c Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 18:02:22 +0900 Subject: [PATCH 87/96] yolo v5 test --- CNN/edge_inference_request_server.py | 61 ++-------------------------- 1 file changed, 3 insertions(+), 58 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index d6ca333..e0ec35b 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -140,7 +140,8 @@ def save_model(model, saved_model_dir): # Run inference yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) -path, im, im0s, vid_cap, s = yolov5_dataset[0] +ims = [im for path, im, im0s, vid_cap, s in yolov5_dataset] +im = ims[0] # for path, im, im0s, vid_cap, s in yolov5_dataset: # with dt[0]: @@ -215,68 +216,12 @@ def inceptionv3(): @app.route('/yolo_v5') def yolov5(): - # for path, im, im0s, vid_cap, s in yolov5_dataset: - # inference_start_time = time.time() result = yolo_model(im) inference_end_time = time.time() - inference_time = inference_end_time - inference_start_time - -# -# inference_start_time = time.time() -# yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' -# weights = './model/yolov5/yolov5s.pt' -# data = './model/yolov5/coco128.yaml' # dataset.yaml path -# imgsz = (640, 640) # inference size (height, width) -# conf_thres = 0.25 # confidence threshold -# iou_thres = 0.45 # NMS IOU threshold -# max_det = 1000 # maximum detections per image -# vid_stride = 1, # video frame-rate stride -# -# yolo_model = DetectMultiBackend(weights, data=data) -# stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt -# imgsz = check_img_size(imgsz, s=stride) # check image size -# -# # Dataloader -# bs = 1 # batch_size -# yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) -# -# # Run inference -# yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup -# seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) -# for path, im, im0s, vid_cap, s in yolov5_dataset: -# with dt[0]: -# im = torch.from_numpy(im).to(yolo_model.device) -# im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 -# im /= 255 # 0 - 255 to 0.0 - 1.0 -# if len(im.shape) == 3: -# im = im[None] # expand for batch dim -# -# # Inference -# with dt[1]: -# pred = yolo_model(im) -# -# # NMS -# with dt[2]: -# pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) -# -# # Process predictions -# for i, det in enumerate(pred): # per image -# seen += 1 -# p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) -# p = Path(p) # to Path -# -# # Print time (inference-only) -# LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") -# -# # Print results -# t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image -# LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) -# inference_end_time = time.time() -# -# inference_time = inference_end_time - inference_start_time return f'yolov5 inference success\ninference time:{inference_time}\n' + app.run(host=hostname, port=port, threaded=False) From 5eaf308cfc28727c4f23630b91de053504a83b85 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 18:16:36 +0900 Subject: [PATCH 88/96] yolo v5 test --- CNN/edge_inference_request_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index e0ec35b..367111f 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -143,6 +143,12 @@ def save_model(model, saved_model_dir): ims = [im for path, im, im0s, vid_cap, s in yolov5_dataset] im = ims[0] +im = torch.from_numpy(im).to(yolo_model.device) +im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 +im /= 255 # 0 - 255 to 0.0 - 1.0 +if len(im.shape) == 3: + im = im[None] # expand for batch dim + # for path, im, im0s, vid_cap, s in yolov5_dataset: # with dt[0]: # im = torch.from_numpy(im).to(yolo_model.device) From 7dd040d3cb4c08a6957a802e57120bc4c3f20868 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 18:22:44 +0900 Subject: [PATCH 89/96] =?UTF-8?q?yolo=20v5=20=EC=B6=94=EA=B0=80=EC=99=84?= =?UTF-8?q?=EB=A3=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 367111f..bfc2f30 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -99,24 +99,23 @@ def save_model(model, saved_model_dir): model.save(saved_model_dir, include_optimizer=False, save_format='tf') -# -# print('\nsaving and loading models...') -# +print('\nsaving and loading models...') + loaded_models = {} -# -# for model_name in models_to_load: -# model_names = models_detail.keys() -# if model_name in model_names: -# model_path = f'{model_name}_saved_model' -# if os.path.isdir(model_path) == False: -# print('model save') -# save_model(model_name, model_path) -# loaded_models[model_name] = tf.keras.models.load_model(model_path) -# else: -# print(f'model names must be in {model_names}') -# exit(1) -# -# print('saving and loading models completed!\n') + +for model_name in models_to_load: + model_names = models_detail.keys() + if model_name in model_names: + model_path = f'{model_name}_saved_model' + if os.path.isdir(model_path) == False: + print('model save') + save_model(model_name, model_path) + loaded_models[model_name] = tf.keras.models.load_model(model_path) + else: + print(f'model names must be in {model_names}') + exit(1) + +print('saving and loading models completed!\n') # Yolo v5 From f4207ec83dda84217f018e0f0efd7aa0706e85fa Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 14 Mar 2023 18:44:27 +0900 Subject: [PATCH 90/96] =?UTF-8?q?=EC=8A=A4=EC=BC=80=EC=A4=84=EB=9F=AC?= =?UTF-8?q?=EC=97=90=20yolo=20v5=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index e1af75f..0a0f0ff 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -26,11 +26,11 @@ # 이 부분만 설정하면 모델추가나 장비추가가 수월함. 각 장비의 ip와 로드된 모델들을 설정해주어야함. edges_info = {'nvidia-xavier2': {'ip_addr': '192.168.0.32', 'ports': [5001, 5002], - 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] + 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3', 'yolo_v5'] }, 'nvidia-tx2': {'ip_addr': '192.168.0.22', 'ports': [5001], - 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3'] + 'models': ['mobilenet', 'mobilenet_v2', 'inception_v3', 'yolo_v5'] }, 'nvidia-nano1': {'ip_addr': '192.168.0.41', 'ports': [5001], From a75fbbb48064533ea231f7baa3c62efd13099e4b Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 16 Mar 2023 12:26:42 +0900 Subject: [PATCH 91/96] =?UTF-8?q?yolo=20v5=EB=A5=BC=20torch=20=EB=8C=80?= =?UTF-8?q?=EC=8B=A0=20tf=EB=A5=BC=20=EC=9D=B4=EC=9A=A9=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EB=B3=91=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index bfc2f30..9e7e820 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -120,7 +120,7 @@ def save_model(model, saved_model_dir): # Yolo v5 yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' -weights = './model/yolov5/yolov5s.pt' +weights = './model/yolov5/yolov5s_saved_model' data = './model/yolov5/coco128.yaml' # dataset.yaml path imgsz = (640, 640) # inference size (height, width) conf_thres = 0.25 # confidence threshold From 4e2a452899d521ece026edcd2d86a9694f9e25d2 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Tue, 21 Mar 2023 20:38:03 +0900 Subject: [PATCH 92/96] edge_inference_request_server.py refactoring --- CNN/edge_inference_request_server.py | 162 +++++++++------------------ 1 file changed, 52 insertions(+), 110 deletions(-) diff --git a/CNN/edge_inference_request_server.py b/CNN/edge_inference_request_server.py index 9e7e820..2b1125c 100644 --- a/CNN/edge_inference_request_server.py +++ b/CNN/edge_inference_request_server.py @@ -1,20 +1,21 @@ import argparse from model.yolov5.models.common import DetectMultiBackend -from model.yolov5.utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams -from model.yolov5.utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, - colorstr, cv2, - increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, - xyxy2xywh) +from model.yolov5.utils.dataloaders import LoadImages +from model.yolov5.utils.general import (Profile, check_img_size) import torch -from pathlib import Path - from flask import Flask import tensorflow as tf import numpy as np import shutil import os import time +from tensorflow.keras.applications import ( + mobilenet, + mobilenet_v2, + inception_v3 +) + gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: @@ -23,14 +24,9 @@ [tf.config.experimental.VirtualDeviceConfiguration( memory_limit=0.6 * 1024)]) -from tensorflow.keras.applications import ( - mobilenet, - mobilenet_v2, - inception_v3 -) parser = argparse.ArgumentParser() -parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3', type=str) +parser.add_argument('--model', default='mobilenet,mobilenet_v2,inception_v3,yolo_v5', type=str) parser.add_argument('--hostname', default='0.0.0.0', type=str) parser.add_argument('--port', default=5001, type=int) args = parser.parse_args() @@ -38,6 +34,8 @@ hostname = args.hostname port = args.port + +# mobilenet, mobilenet v2, inception v3 models = { 'mobilenet': mobilenet, 'mobilenet_v2': mobilenet_v2, @@ -51,26 +49,16 @@ } -def mobilenet_load_image(image_path): - return tf.keras.preprocessing.image.load_img( - image_path, - target_size=[224, 224]) - - -def inception_load_image(image_path): - return tf.keras.preprocessing.image.load_img( - image_path, - target_size=[299, 299]) - +def preprocess_image(image_path, model, target_size): + test_image = tf.keras.preprocessing.image.load_img(image_path, target_size=target_size) + test_image_array = tf.keras.preprocessing.image.img_to_array(test_image, dtype=np.int32) + return model.preprocess_input(test_image_array[tf.newaxis, ...]) -def image_to_array(image): - return tf.keras.preprocessing.image.img_to_array(image, dtype=np.int32) - - -def image_preprocess(image_array, model_name): - return models[model_name].preprocess_input( - image_array[tf.newaxis, ...]) +def save_model(model, saved_model_dir): + model = models_detail[model] + shutil.rmtree(saved_model_dir, ignore_errors=True) + model.save(saved_model_dir, include_optimizer=False, save_format='tf') print('\npreprossing images...') @@ -78,27 +66,12 @@ def image_preprocess(image_array, model_name): mobilenetv2_image_path = './dataset/imagenet/imagenet_1000_raw/n04404412_1.JPEG' inceptionv3_image_path = './dataset/imagenet/imagenet_1000_raw/n13040303_1.JPEG' -mobilenetv1_test_image = mobilenet_load_image(mobilenetv1_image_path) -mobilenetv2_test_image = mobilenet_load_image(mobilenetv2_image_path) -inceptionv3_test_image = inception_load_image(inceptionv3_image_path) - -mobilenetv1_test_image_array = image_to_array(mobilenetv1_test_image) -mobilenetv2_test_image_array = image_to_array(mobilenetv2_test_image) -inceptionv3_test_image_array = image_to_array(inceptionv3_test_image) - -mobilenetv1_test_image_preprocessed = image_preprocess(mobilenetv1_test_image_array, 'mobilenet') -mobilenetv2_test_image_preprocessed = image_preprocess(mobilenetv2_test_image_array, 'mobilenet_v2') -inceptionv3_test_image_preprocessed = image_preprocess(inceptionv3_test_image_array, 'inception_v3') +mobilenetv1_test_image_preprocessed = preprocess_image(mobilenetv1_image_path, mobilenet, [224, 224]) +mobilenetv2_test_image_preprocessed = preprocess_image(mobilenetv2_image_path, mobilenet_v2, [224, 224]) +inceptionv3_test_image_preprocessed = preprocess_image(inceptionv3_image_path, inception_v3, [299, 299]) print('image preprocessing completed!\n') - -def save_model(model, saved_model_dir): - model = models_detail[model] - shutil.rmtree(saved_model_dir, ignore_errors=True) - model.save(saved_model_dir, include_optimizer=False, save_format='tf') - - print('\nsaving and loading models...') loaded_models = {} @@ -108,75 +81,44 @@ def save_model(model, saved_model_dir): if model_name in model_names: model_path = f'{model_name}_saved_model' if os.path.isdir(model_path) == False: - print('model save') + print('\nmodel save!\n') save_model(model_name, model_path) loaded_models[model_name] = tf.keras.models.load_model(model_path) else: - print(f'model names must be in {model_names}') - exit(1) + continue print('saving and loading models completed!\n') # Yolo v5 -yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' -weights = './model/yolov5/yolov5s_saved_model' -data = './model/yolov5/coco128.yaml' # dataset.yaml path -imgsz = (640, 640) # inference size (height, width) -conf_thres = 0.25 # confidence threshold -iou_thres = 0.45 # NMS IOU threshold -max_det = 1000 # maximum detections per image -vid_stride = 1, # video frame-rate stride - -yolo_model = DetectMultiBackend(weights, data=data) -stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt -imgsz = check_img_size(imgsz, s=stride) # check image size - -# Dataloader -bs = 1 # batch_size -yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) - -# Run inference -yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup -seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) -ims = [im for path, im, im0s, vid_cap, s in yolov5_dataset] -im = ims[0] - -im = torch.from_numpy(im).to(yolo_model.device) -im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 -im /= 255 # 0 - 255 to 0.0 - 1.0 -if len(im.shape) == 3: - im = im[None] # expand for batch dim - -# for path, im, im0s, vid_cap, s in yolov5_dataset: -# with dt[0]: -# im = torch.from_numpy(im).to(yolo_model.device) -# im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 -# im /= 255 # 0 - 255 to 0.0 - 1.0 -# if len(im.shape) == 3: -# im = im[None] # expand for batch dim -# -# # Inference -# with dt[1]: -# pred = yolo_model(im) -# -# # NMS -# with dt[2]: -# pred = non_max_suppression(pred, conf_thres, iou_thres, max_det=max_det) -# -# # Process predictions -# for i, det in enumerate(pred): # per image -# seen += 1 -# p, im0, frame = path, im0s.copy(), getattr(yolov5_dataset, 'frame', 0) -# p = Path(p) # to Path -# -# # Print time (inference-only) -# LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") -# -# -# # Print results -# t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image -# LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) +if 'yolo_v5' in models_to_load: + yolov5_image_path = './dataset/imagenet/imagenet_1000_raw/n02782093_1.JPEG' + weights = './model/yolov5/yolov5s_saved_model' + data = './model/yolov5/coco.yaml' # dataset.yaml path + imgsz = (640, 640) # inference size (height, width) + conf_thres = 0.25 # confidence threshold + iou_thres = 0.45 # NMS IOU threshold + max_det = 1000 # maximum detections per image + vid_stride = 1, # video frame-rate stride + + yolo_model = DetectMultiBackend(weights, data=data) + stride, names, pt = yolo_model.stride, yolo_model.names, yolo_model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + yolov5_dataset = LoadImages(yolov5_image_path, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + + yolo_model.warmup(imgsz=(1 if pt or yolo_model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + ims = [im for path, im, im0s, vid_cap, s in yolov5_dataset] + im = ims[0] + + im = torch.from_numpy(im).to(yolo_model.device) + im = im.half() if yolo_model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim app = Flask(__name__) From 773bbae745307299e8c6311a2c3283cf7b70c58d Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 23 Mar 2023 13:13:53 +0900 Subject: [PATCH 93/96] =?UTF-8?q?=EC=B6=94=EB=A1=A0=20=EA=B2=B0=EA=B3=BC?= =?UTF-8?q?=EB=A1=9C=20=EC=9D=91=EB=8B=B5=EC=8B=9C=EA=B0=84=EC=9D=84=20?= =?UTF-8?q?=EB=B3=BC=20=EC=88=98=20=EC=9E=88=EA=B2=8C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/edge_inference_request_scheduler.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CNN/edge_inference_request_scheduler.py b/CNN/edge_inference_request_scheduler.py index 0a0f0ff..4bbb62d 100644 --- a/CNN/edge_inference_request_scheduler.py +++ b/CNN/edge_inference_request_scheduler.py @@ -144,6 +144,7 @@ def model_request(edge, model, order): inference_time = res.text.split(':')[1] inference_time = inference_time.split('\n')[0] inference_time_results[order-1] = float(inference_time) + request_time_results[order-1] = float(processing_time) print(f'[{order}:{edge}({port})/{model}] total request time: {processing_time}\n{res.text}') return @@ -168,6 +169,7 @@ def model_request(edge, model, order): order = 0 inference_time_results = [0 for _ in range(len(requests_list))] +request_time_results = [0 for _ in range(len(requests_list))] request_sleep_time = 1 / len(requests_list) # 요청들을 1초에 나눠서 보내기 위한 슬립시간 for req in requests_list: @@ -189,6 +191,8 @@ def model_request(edge, model, order): # 추론요청 결과 출력 (최소, 중간, 최대, 평균) inference_time_results.sort() len_inference_time_results = len(inference_time_results) +request_time_results.sort() +len_request_time_results = len(request_time_results) total_inference_time = sum(inference_time_results) avg_inference_time = total_inference_time / len_inference_time_results @@ -196,9 +200,18 @@ def model_request(edge, model, order): mid_inference_time = inference_time_results[int(len_inference_time_results / 2)] max_inference_time = inference_time_results[-1] +total_request_time = sum(request_time_results) +avg_request_time = total_request_time / len_request_time_results +min_request_time = request_time_results[0] +mid_request_time = request_time_results[int(len_request_time_results / 2)] +max_request_time = request_time_results[-1] + print(f'평균 추론 시간: {avg_inference_time}') print(f'최소 추론 시간: {min_inference_time}') print(f'중간 추론 시간: {mid_inference_time}') print(f'최대 추론 시간: {max_inference_time}\n') - +print(f'평균 응답 시간: {avg_request_time}') +print(f'최소 응답 시간: {min_request_time}') +print(f'중간 응답 시간: {mid_request_time}') +print(f'최대 응답 시간: {max_request_time}\n') From 2b111e0c0a469e87bf4726cf0be4799c5c923744 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 30 Mar 2023 23:20:50 +0900 Subject: [PATCH 94/96] =?UTF-8?q?edge=20=EC=9E=A5=EB=B9=84=EC=97=90?= =?UTF-8?q?=EC=84=9C=20tf=20serving=EC=9D=84=20=EC=9C=84=ED=95=9C=20Docker?= =?UTF-8?q?file,=20model=5Fconfig=5Flist=20file,=20docker=5Frun=5Ftf=5Fser?= =?UTF-8?q?ving.sh=20file=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/Dockerfile | 5 +++++ CNN/docker_run_tf_serving.sh | 12 ++++++++++++ CNN/model/models.config | 27 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 CNN/Dockerfile create mode 100644 CNN/docker_run_tf_serving.sh create mode 100644 CNN/model/models.config diff --git a/CNN/Dockerfile b/CNN/Dockerfile new file mode 100644 index 0000000..f3d02f8 --- /dev/null +++ b/CNN/Dockerfile @@ -0,0 +1,5 @@ +FROM helmuthva/jetson-xavier-tensorflow-serving-base + +EXPOSE 8500 + +CMD ["tensorflow_model_server", "--port=8500", "--model_config_file=/models/model/models.config", "--grpc_channel_arguments=grpc.max_send_message_length=50*1024*1024", "--grpc_channel_arguments=grpc.max_receive_length=50*1024*1024", "--grpc_max_threads=1000"] \ No newline at end of file diff --git a/CNN/docker_run_tf_serving.sh b/CNN/docker_run_tf_serving.sh new file mode 100644 index 0000000..62f5bed --- /dev/null +++ b/CNN/docker_run_tf_serving.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +docker run --rm \ + --device /dev/nvhost-ctrl \ + --device /dev/nvhost-ctrl-gpu \ + --device /dev/nvhost-prof-gpu \ + --device /dev/nvmap \ + --device /dev/nvhost-gpu \ + --device /dev/nvhost-as-gpu \ + -p 8500:8500 \ + -v ~/edge-inference/CNN/model/:/models/model/ \ + edge-tf-serving:latest \ No newline at end of file diff --git a/CNN/model/models.config b/CNN/model/models.config new file mode 100644 index 0000000..8453ba4 --- /dev/null +++ b/CNN/model/models.config @@ -0,0 +1,27 @@ +# models.config + +model_config_list { + config { + name : "mobilenet_v1" + base_path: "/models/model/mobilenet_v1/" + model_platform: "tensorflow" + } + + config { + name: "mobilenet_v2" + base_path: "/models/model/mobilenet_v2/" + model_platform: "tensorflow" + } + + config { + name: "inception_v3" + base_path: "/models/model/inception_v3/" + model_platform: "tensorflow" + } + +# config { +# name: "yolo_v5" +# base_path: "models/model/yolo_v5/" +# model_platform: "tensorflow" +# } +} \ No newline at end of file From 75ea23221c384c21452f0cbc05ea22c79b8394b1 Mon Sep 17 00:00:00 2001 From: kh3654po Date: Thu, 30 Mar 2023 23:25:45 +0900 Subject: [PATCH 95/96] =?UTF-8?q?edge=20=EC=9E=A5=EB=B9=84=EC=97=90?= =?UTF-8?q?=EC=84=9C=20tf=20serving=EC=9D=84=20=EC=9C=84=ED=95=9C=20?= =?UTF-8?q?=EC=9D=B4=EB=AF=B8=EC=A7=80=20=EB=B9=8C=EB=93=9C=20shell=20scri?= =?UTF-8?q?pt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/docker_build_tf_serving.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CNN/docker_build_tf_serving.sh diff --git a/CNN/docker_build_tf_serving.sh b/CNN/docker_build_tf_serving.sh new file mode 100644 index 0000000..82955f6 --- /dev/null +++ b/CNN/docker_build_tf_serving.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t edge-tf-serving . \ No newline at end of file From 6ca0c8c1b77ef4ac9d204da4ef78745023459f7c Mon Sep 17 00:00:00 2001 From: kh3654po Date: Fri, 31 Mar 2023 21:05:29 +0900 Subject: [PATCH 96/96] =?UTF-8?q?tf=20serving=EB=A5=BC=20=EC=9C=84?= =?UTF-8?q?=ED=95=9C=20docker=20build/run=20bash=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CNN/run_tf_serving_container.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 CNN/run_tf_serving_container.sh diff --git a/CNN/run_tf_serving_container.sh b/CNN/run_tf_serving_container.sh new file mode 100644 index 0000000..c891448 --- /dev/null +++ b/CNN/run_tf_serving_container.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +is_dockerfile=$(ls Dockerfile) + +if [ -z $is_dockerfile ] +then + echo "Dockerfile not exist!" + exit +fi + +image_name="edge-tf-serving" +image_id=$(docker images -aq $image_name) + +if [ -z "$image_id" ] +then + docker build -t edge-tf-serving . +fi + +docker run --rm \ + --device /dev/nvhost-ctrl \ + --device /dev/nvhost-ctrl-gpu \ + --device /dev/nvhost-prof-gpu \ + --device /dev/nvmap \ + --device /dev/nvhost-gpu \ + --device /dev/nvhost-as-gpu \ + -p 8500:8500 \ + -v ~/edge-inference/CNN/model/:/models/model/ \ + edge-tf-serving:latest \ No newline at end of file