setup dvc

nthaihoc · nthaihoc · commit 10d139d4f237 · 2024-09-28T13:31:33.000+07:00
diff --git a/.dvc/.gitignore b/.dvc/.gitignore
@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache
diff --git a/.dvc/config b/.dvc/config
diff --git a/.dvcignore b/.dvcignore
@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore
diff --git a/dvc.lock b/dvc.lock
@@ -0,0 +1,79 @@
+schema: '2.0'
+stages:
+  data_ingestion:
+    cmd: python3 src/ccs/pipeline/stage_01_data_ingestion.py
+    deps:
+    - path: config/config.yaml
+      hash: md5
+      md5: 9380aa36e63440ed993bb48e759891e3
+      size: 873
+    - path: src/ccs/pipeline/stage_01_data_ingestion.py
+      hash: md5
+      md5: 37148056a99fe2812e148b0fc6dc6ee2
+      size: 851
+    outs:
+    - path: artifacts/data_ingestion/dataset_final
+      hash: md5
+      md5: fb8c0835550d475bdc6d20ac00304272.dir
+      size: 2402769676
+      nfiles: 22434
+  prepared_model:
+    cmd: python3 src/ccs/pipeline/stage_02_prepare_model.py
+    deps:
+    - path: config/config.yaml
+      hash: md5
+      md5: 9380aa36e63440ed993bb48e759891e3
+      size: 873
+    - path: params.yaml
+      hash: md5
+      md5: 5ebd37d7e1d428cf2d85caa6dcc4edcc
+      size: 161
+    - path: src/ccs/pipeline/stage_02_prepare_model.py
+      hash: md5
+      md5: a72369570adbf3cfd71d5333702bc6d5
+      size: 838
+    outs:
+    - path: artifacts/model/model_train.keras
+      hash: md5
+      md5: d796e2658891a6255829c3ebd397f8e1
+      size: 13075888
+  training_model:
+    cmd: python3 src/ccs/pipeline/stage_03_training_model.py
+    deps:
+    - path: config/config.yaml
+      hash: md5
+      md5: 9380aa36e63440ed993bb48e759891e3
+      size: 873
+    - path: params.yaml
+      hash: md5
+      md5: 5ebd37d7e1d428cf2d85caa6dcc4edcc
+      size: 161
+    - path: src/ccs/pipeline/stage_03_training_model.py
+      hash: md5
+      md5: 422dac12a5b9a82823d9dba7d97e8fd6
+      size: 776
+    outs:
+    - path: artifacts/model/model_trained.keras
+      hash: md5
+      md5: b29e93b9d809808ddda2c1c6b67252ee
+      size: 19985337
+  evaluate_model:
+    cmd: python3 src/ccs/pipeline/stage_04_evaluate_model.py
+    deps:
+    - path: config/config.yaml
+      hash: md5
+      md5: 9380aa36e63440ed993bb48e759891e3
+      size: 873
+    - path: params.yaml
+      hash: md5
+      md5: 5ebd37d7e1d428cf2d85caa6dcc4edcc
+      size: 161
+    - path: src/ccs/pipeline/stage_04_evaluate_model.py
+      hash: md5
+      md5: 0ebd6a6f0bba907b99a0e3441095ce66
+      size: 873
+    outs:
+    - path: scores.json
+      hash: md5
+      md5: 4d86cd8e65758898762a8f7ee4c21910
+      size: 79
diff --git a/dvc.yaml b/dvc.yaml
@@ -0,0 +1,36 @@
+stages:
+  data_ingestion:
+    cmd: python3 src/ccs/pipeline/stage_01_data_ingestion.py
+    deps:
+      - src/ccs/pipeline/stage_01_data_ingestion.py
+      - config/config.yaml
+    outs:
+      - artifacts/data_ingestion/dataset_final
+    
+  prepared_model:
+    cmd: python3 src/ccs/pipeline/stage_02_prepare_model.py
+    deps:
+      - src/ccs/pipeline/stage_02_prepare_model.py
+      - config/config.yaml
+      - params.yaml
+    outs:
+      - artifacts/model/model_train.keras
+  
+  training_model:
+    cmd: python3 src/ccs/pipeline/stage_03_training_model.py
+    deps:
+      - src/ccs/pipeline/stage_03_training_model.py
+      - config/config.yaml
+      - params.yaml
+    outs:
+      - artifacts/model/model_trained.keras
+    
+  evaluate_model:
+    cmd: python3 src/ccs/pipeline/stage_04_evaluate_model.py
+    deps:
+      - src/ccs/pipeline/stage_04_evaluate_model.py
+      - config/config.yaml
+      - params.yaml
+    outs:
+      - scores.json:
+          cache: false
diff --git a/main.py b/main.py
@@ -8,15 +8,15 @@
 import numpy as np
 from pathlib import Path
 
-# STAGE_NAME = "Data Ingestion Stage"
-# try:
-#     logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
-#     data_ingestion = DataIngestionPipeline()
-#     data_ingestion.main()
-#     logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
-# except Exception as e:
-#     logger.exception(e)
-#     raise e
+STAGE_NAME = "Data Ingestion Stage"
+try:
+    logger.info(f">>>>> stage {STAGE_NAME} started <<<<<")
+    data_ingestion = DataIngestionPipeline()
+    data_ingestion.main()
+    logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
+except Exception as e:
+    logger.exception(e)
+    raise e
 
 STAGE_NAME = "Prepare Model"
 try:
@@ -47,6 +47,4 @@
     logger.info(f">>>>> stage {STAGE_NAME} completed <<<<<")
 except Exception as e:
     logger.exception(e)
-    raise e
-
-
+    raise e
diff --git a/params.yaml b/params.yaml
@@ -1,10 +1,10 @@
 IMAGE_SIZE: [224, 224, 3]
 BATCH_SIZE: 16
 INCLUDE_TOP: False
-EPOCHS: 10
+EPOCHS: 15
 CLASSES: 5
 WEIGHTS: imagenet
-LEARNING_RATE: 0.0001
+LEARNING_RATE: 0.001
 BETA_1: 0.9
 BETA_2: 0.999
 DECAY: 0.0001
diff --git a/src/ccs/components/data_ingestion.py b/src/ccs/components/data_ingestion.py
@@ -14,7 +14,6 @@ def download_file(self) -> str:
         try:
             dataset_url = self.config.data_URL 
             zip_download_dir = self.config.local_data_file
-            #os.makedirs("artifacts/data_ingestion", exist_ok=True)
             logger.info(f"Download data from {dataset_url} into file {zip_download_dir}")
 
             file_id = dataset_url.split("/")[-2]
@@ -28,7 +27,6 @@ def download_file(self) -> str:
     
     def extract_zip_file(self):
         unzip_path = self.config.unzip_dir
-        os.makedirs(unzip_path, exist_ok=True)
         with zipfile.ZipFile(self.config.local_data_file, "r") as zip_ref:
             zip_ref.extractall(unzip_path)
 
diff --git a/src/ccs/config/configuration.py b/src/ccs/config/configuration.py
@@ -15,13 +15,12 @@ def __init__(
         self.config = read_yaml(config_filepath)
         self.params = read_yaml(params_filepath)
 
-        #create_directories([self.config.artifacts_root])
+        create_directories([self.config.artifacts_root])
 
-    
     def get_data_ingestion_config(self) -> DataIngestionConfig:
         config = self.config.data_ingestion
 
-        #create_directories([config.root_dir])
+        create_directories([config.root_dir])
         
         data_ingestion_config = DataIngestionConfig(
             root_dir=config.root_dir,
diff --git a/src/ccs/pipeline/stage_01_data_ingestion.py b/src/ccs/pipeline/stage_01_data_ingestion.py
@@ -2,7 +2,7 @@
 from ccs.components.data_ingestion import DataIngestion
 from ccs import logger
 
-STAGE_NAME = "Data Ingestion"
+STAGE_NAME = "DATA INGESTION"
 
 class DataIngestionPipeline:
     def __init__(self):
@@ -17,10 +17,10 @@ def main(self):
 
 if __name__ == "__main__":
     try:
-        logger.info(f">>>>> Stage {STAGE_NAME} started <<<<<")
+        logger.info(f">>>>>> Stage {STAGE_NAME} started <<<<<<")
         data_ingestion = DataIngestionPipeline()
         data_ingestion.main()
-        logger.info(f">>>>> Stage {STAGE_NAME} completed <<<<<")
+        logger.info(f">>>>>> Stage {STAGE_NAME} completed <<<<<<\n\n x============x")
     except Exception as e:
         logger.exception(e)
         raise e
diff --git a/src/ccs/pipeline/stage_02_prepare_model.py b/src/ccs/pipeline/stage_02_prepare_model.py
@@ -3,6 +3,8 @@
 from ccs import logger
 
 
+STAGE_NAME = "PREPARE MODEL"
+
 class PrepareModelPipeline:
     def __int__(self):
         pass
@@ -14,3 +16,12 @@ def main(self):
         model = prepare_model.base_model()
         model = prepare_model.full_model()
 
+if __name__ == "__main__":
+    try:
+        logger.info(f">>>>>> Stage {STAGE_NAME} started <<<<<<")
+        prepare_model = PrepareModelPipeline()
+        prepare_model.main()
+        logger.info(f">>>>>> Stage {STAGE_NAME} completed <<<<<<\n\n x============x")
+    except Exception as e:
+        logger.exception(e)
+        raise e
diff --git a/src/ccs/pipeline/stage_03_training_model.py b/src/ccs/pipeline/stage_03_training_model.py
@@ -2,6 +2,7 @@
 from ccs.components.model_training import TrainingModel
 from ccs.config.configuration import ConfigurationManager
 
+STAGE_NAME = "TRAINING MODEL"
 
 class TrainModelPipeline:
     def __init__ (self):
@@ -12,4 +13,14 @@ def main(self):
         prepare_config = config.setup_train_model()
         model_train = TrainingModel(config=prepare_config)
         model_train.train_model()
+
+if __name__ == "__main__":
+    try:
+        logger.info(f">>>>>> Stage {STAGE_NAME} started <<<<<<")
+        train_model = TrainModelPipeline()
+        train_model.main()
+        logger.info(f">>>>>> Stage {STAGE_NAME} completed <<<<<<\n\n x============x")
+    except Exception as e:
+        logger.exception(e)
+        raise e
         
diff --git a/src/ccs/pipeline/stage_04_evaluate_model.py b/src/ccs/pipeline/stage_04_evaluate_model.py
@@ -2,6 +2,7 @@
 from ccs.components.evaluate_model import EvaluateModel
 from ccs.config.configuration import ConfigurationManager
 
+STAGE_NAME = "EVALUATION MODEL"
 
 class EvaluateModelPipeline:
     def __init__ (self):
@@ -13,4 +14,15 @@ def main(self):
         evaluate_model = EvaluateModel(config=evaluate_config)
         evaluate_model.evaluation()
         evaluate_model.save_score()
-        evaluate_model.log_into_mlflow()
+        evaluate_model.log_into_mlflow()
+    
+
+if __name__ == "__main__":
+    try:
+        logger.info(f">>>>>> Stage {STAGE_NAME} started <<<<<<")
+        evaluate_model = EvaluateModelPipeline()
+        evaluate_model.main()
+        logger.info(f">>>>>> Stage {STAGE_NAME} completed <<<<<<\n\n x============x")
+    except Exception as e:
+        logger.exception(e)
+        raise e

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+# Add patterns of files dvc should ignore, which could improve`
	`2`	`+# the performance. Learn more at`
	`3`	`+# https://dvc.org/doc/user-guide/dvcignore`