Added new data

0xShun · 0xShun · commit 94d9f0e36199 · 2024-05-20T00:13:01.000+08:00
diff --git a/STS_Research/configs.py b/STS_Research/configs.py
@@ -0,0 +1,17 @@
+import os
+from datetime import datetime
+
+from mltu.configs import BaseModelConfigs
+
+class ModelConfigs(BaseModelConfigs):
+    def __init__(self):
+        super().__init__()
+        self.model_path = os.path.join("Models/04_sentence_recognition", datetime.strftime(datetime.now(), "%Y%m%d%H%M"))
+        self.vocab = ""
+        self.height = 96
+        self.width = 1408
+        self.max_text_length = 0
+        self.batch_size = 32
+        self.learning_rate = 0.0005
+        self.train_epochs = 1000
+        self.train_workers = 20
diff --git a/STS_Research/inferenceModel.py b/STS_Research/inferenceModel.py
@@ -0,0 +1,56 @@
+import cv2
+import typing
+import numpy as np
+
+from mltu.inferenceModel import OnnxInferenceModel
+from mltu.utils.text_utils import ctc_decoder, get_cer, get_wer
+from mltu.transformers import ImageResizer
+
+class ImageToWordModel(OnnxInferenceModel):
+    def __init__(self, char_list: typing.Union[str, list], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.char_list = char_list
+
+    def predict(self, image: np.ndarray):
+        image = ImageResizer.resize_maintaining_aspect_ratio(image, *self.input_shapes[0][1:3][::-1])
+
+        image_pred = np.expand_dims(image, axis=0).astype(np.float32)
+
+        preds = self.model.run(self.output_names, {self.input_names[0]: image_pred})[0]
+
+        text = ctc_decoder(preds, self.char_list)[0]
+
+        return text
+
+if __name__ == "__main__":
+    import pandas as pd
+    from tqdm import tqdm
+    from mltu.configs import BaseModelConfigs
+
+    configs = BaseModelConfigs.load("Models/04_sentence_recognition/202301131202/configs.yaml")
+
+    model = ImageToWordModel(model_path=configs.model_path, char_list=configs.vocab)
+
+    df = pd.read_csv("Models/04_sentence_recognition/202301131202/val.csv").values.tolist()
+
+    accum_cer, accum_wer = [], []
+    for image_path, label in tqdm(df):
+        image = cv2.imread(image_path.replace("\\", "/"))
+
+        prediction_text = model.predict(image)
+
+        cer = get_cer(prediction_text, label)
+        wer = get_wer(prediction_text, label)
+        print("Image: ", image_path)
+        print("Label:", label)
+        print("Prediction: ", prediction_text)
+        print(f"CER: {cer}; WER: {wer}")
+
+        accum_cer.append(cer)
+        accum_wer.append(wer)
+
+        cv2.imshow(prediction_text, image)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
+    print(f"Average CER: {np.average(accum_cer)}, Average WER: {np.average(accum_wer)}")
diff --git a/STS_Research/login.html b/STS_Research/login.html
@@ -0,0 +1,127 @@
+
+
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <title>Kaggle: Your Home for Data Science</title>
+  <meta charset="utf-8" />
+    <meta name="robots" content="index, follow" />
+  <meta name="description" content="Kaggle is the world&#x2019;s largest data science community with powerful tools and resources to help you achieve your data science goals." />
+  <meta name="turbolinks-cache-control" content="no-cache" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, minimum-scale=1.0">
+  <meta name="theme-color" content="#008ABC" />
+  <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" type="text/javascript">
+    window["pageRequestStartTime"] = 1716135025964;
+    window["pageRequestEndTime"] = 1716135025968;
+    window["initialPageLoadStartTime"] = new Date().getTime();
+  </script>
+  <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" id="gsi-client" src="https://accounts.google.com/gsi/client" async defer></script>
+  <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==">window.KAGGLE_JUPYTERLAB_PATH = "/static/assets/jupyterlab/jupyterlab-index-9369516b66d9c0a10de7.html";</script>
+  <link rel="preconnect" href="https://www.google-analytics.com" crossorigin="anonymous" /><link rel="preconnect" href="https://stats.g.doubleclick.net" /><link rel="preconnect" href="https://storage.googleapis.com" /><link rel="preconnect" href="https://apis.google.com" />
+  <link href="/static/images/favicon.ico" rel="shortcut icon" type="image/x-icon" />
+  <link rel="manifest" href="/static/json/manifest.json" crossorigin="use-credentials">
+
+
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+
+  <link href="https://fonts.googleapis.com/css?family=Inter:400,400i,500,500i,600,600i,700,700i&display=swap"
+    rel="preload" as="style" />
+  <link href="https://fonts.googleapis.com/css2?family=Google+Symbols:FILL@0..1&display=block"
+    rel="preload" as="style" />
+  <link href="https://fonts.googleapis.com/css?family=Inter:400,400i,500,500i,600,600i,700,700i&display=swap"
+    rel="stylesheet" media="print" id="async-google-font-1" />
+  <link href="https://fonts.googleapis.com/css2?family=Google+Symbols:FILL@0..1&display=block"
+    rel="stylesheet" media="print" id="async-google-font-2" />
+  <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" type="text/javascript">
+    const styleSheetIds = ["async-google-font-1", "async-google-font-2"];
+    styleSheetIds.forEach(function (id) {
+      document.getElementById(id).addEventListener("load", function() {
+        this.media = "all";
+      });
+    });
+  </script>
+
+  <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" src="https://www.google.com/recaptcha/enterprise.js?render=6LcW02cpAAAAAJlaJemsQQEwAiTEYB4aR6FYE_rD&waf=session" async defer></script>
+   <style>.grecaptcha-badge { visibility: hidden; }</style>
+
+    <link rel="stylesheet" type="text/css" href="/static/assets/vendor.css?v=dne" />
+    <link rel="stylesheet" type="text/css" href="/static/assets/app.css?v=62d595175a876550f3e6" />
+
+  
+    
+ 
+      <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==">
+        try{(function(a,s,y,n,c,h,i,d,e){d=s.createElement("style");
+        d.appendChild(s.createTextNode(""));s.head.appendChild(d);d=d.sheet;
+        y=y.map(x => d.insertRule(x + "{ opacity: 0 !important }"));
+        h.start=1*new Date;h.end=i=function(){y.forEach(x => x<d.cssRules.length ? d.deleteRule(x) : {})};
+        (a[n]=a[n]||[]).hide=h;setTimeout(function(){i();h.end=null},c);h.timeout=c;
+        })(window,document,['.site-header-react__nav'],'dataLayer',2000,{'GTM-52LNT9S':true});}catch(ex){}
+    </script>
+    <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==">
+        window.dataLayer = window.dataLayer || [];
+        function gtag() { dataLayer.push(arguments); }
+        gtag('js', new Date());
+        gtag('config', 'G-T7QHS60L4Q', {
+            'optimize_id': 'GTM-52LNT9S',
+            'displayFeaturesTask': null,
+            'send_page_view': false,
+            'content_group1': 'Account'
+        });
+    </script>
+    <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" async src="https://www.googletagmanager.com/gtag/js?id=G-T7QHS60L4Q"></script>
+
+  
+    
+
+  <meta name="twitter:site" content="@Kaggle" /> 
+  
+    
+
+  
+    
+
+  
+    
+
+
+    <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==">window['useKaggleAnalytics'] = true;</script>
+
+  <script id="gapi-target" nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" src="https://apis.google.com/js/api.js" defer
+    async></script>
+  <script nonce="EiRBuHfyGAC/q+OmJ6g0zQ==" src="/static/assets/runtime.js?v=96c24c08a967efb5ee0c" data-turbolinks-track="reload"></script>
+  <script nonce="EiRBuHfyGAC/q+OmJ6g0zQ==" src="/static/assets/vendor.js?v=38b5f1af3c791be446c0" data-turbolinks-track="reload"></script>
+  <script nonce="EiRBuHfyGAC/q+OmJ6g0zQ==" src="/static/assets/app.js?v=0cb512098feb97aac492" data-turbolinks-track="reload"></script>
+    <script nonce="EiRBuHfyGAC/q&#x2B;OmJ6g0zQ==" type="text/javascript">
+      window.kaggleStackdriverConfig = {
+        key: 'AIzaSyA4eNqUdRRskJsCZWVz-qL655Xa5JEMreE',
+        projectId: 'kaggle-161607',
+        service: 'web-fe',
+        version: 'ci',
+        userId: '0'
+      }
+    </script>
+</head>
+
+<body data-turbolinks="false">
+  <main>
+    
+
+
+
+
+
+<div id="site-container"></div>
+
+<div id="site-body" class="hide">
+    
+</div>
+
+
+
+
+  </main>
+</body>
+
+</html>
diff --git a/STS_Research/model.py b/STS_Research/model.py
@@ -0,0 +1,39 @@
+from keras import layers
+from keras.models import Model
+
+from mltu.tensorflow.model_utils import residual_block
+
+
+def train_model(input_dim, output_dim, activation="leaky_relu", dropout=0.2):
+    
+    inputs = layers.Input(shape=input_dim, name="input")
+
+    # normalize images here instead in preprocessing step
+    input = layers.Lambda(lambda x: x / 255)(inputs)
+
+    x1 = residual_block(input, 32, activation=activation, skip_conv=True, strides=1, dropout=dropout)
+
+    x2 = residual_block(x1, 32, activation=activation, skip_conv=True, strides=2, dropout=dropout)
+    x3 = residual_block(x2, 32, activation=activation, skip_conv=False, strides=1, dropout=dropout)
+
+    x4 = residual_block(x3, 64, activation=activation, skip_conv=True, strides=2, dropout=dropout)
+    x5 = residual_block(x4, 64, activation=activation, skip_conv=False, strides=1, dropout=dropout)
+
+    x6 = residual_block(x5, 128, activation=activation, skip_conv=True, strides=2, dropout=dropout)
+    x7 = residual_block(x6, 128, activation=activation, skip_conv=True, strides=1, dropout=dropout)
+
+    x8 = residual_block(x7, 128, activation=activation, skip_conv=True, strides=2, dropout=dropout)
+    x9 = residual_block(x8, 128, activation=activation, skip_conv=False, strides=1, dropout=dropout)
+
+    squeezed = layers.Reshape((x9.shape[-3] * x9.shape[-2], x9.shape[-1]))(x9)
+
+    blstm = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(squeezed)
+    blstm = layers.Dropout(dropout)(blstm)
+
+    blstm = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(blstm)
+    blstm = layers.Dropout(dropout)(blstm)
+
+    output = layers.Dense(output_dim + 1, activation="softmax", name="output")(blstm)
+
+    model = Model(inputs=inputs, outputs=output)
+    return model
diff --git a/STS_Research/train.py b/STS_Research/train.py
@@ -0,0 +1,122 @@
+import tensorflow as tf
+try: [tf.config.experimental.set_memory_growth(gpu, True) for gpu in tf.config.experimental.list_physical_devices("GPU")]
+except: pass
+
+from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, TensorBoard
+
+from mltu.preprocessors import ImageReader
+from mltu.transformers import ImageResizer, LabelIndexer, LabelPadding, ImageShowCV2
+from mltu.augmentors import RandomBrightness, RandomRotate, RandomErodeDilate, RandomSharpen
+from mltu.annotations.images import CVImage
+
+from mltu.tensorflow.dataProvider import DataProvider
+from mltu.tensorflow.losses import CTCloss
+from mltu.tensorflow.callbacks import Model2onnx, TrainLogger
+from mltu.tensorflow.metrics import CERMetric, WERMetric
+
+from model import train_model
+from configs import ModelConfigs
+
+import os
+from tqdm import tqdm
+
+# Must download and extract datasets manually from https://fki.tic.heia-fr.ch/databases/download-the-iam-handwriting-database to Datasets\IAM_Sentences
+sentences_txt_path = os.path.join("Datasets", "IAM_Sentences", "ascii", "sentences.txt")
+sentences_folder_path = os.path.join("Datasets", "IAM_Sentences", "sentences")
+
+dataset, vocab, max_len = [], set(), 0
+words = open(sentences_txt_path, "r").readlines()
+for line in tqdm(words):
+    if line.startswith("#"):
+        continue
+
+    line_split = line.split(" ")
+    if line_split[2] == "err":
+        continue
+
+    folder1 = line_split[0][:3]
+    folder2 = "-".join(line_split[0].split("-")[:2])
+    file_name = line_split[0] + ".png"
+    label = line_split[-1].rstrip("\n")
+
+    # replace "|" with " " in label
+    label = label.replace("|", " ")
+
+    rel_path = os.path.join(sentences_folder_path, folder1, folder2, file_name)
+    if not os.path.exists(rel_path):
+        print(f"File not found: {rel_path}")
+        continue
+
+    dataset.append([rel_path, label])
+    vocab.update(list(label))
+    max_len = max(max_len, len(label))
+
+# Create a ModelConfigs object to store model configurations
+configs = ModelConfigs()
+
+# Save vocab and maximum text length to configs
+configs.vocab = "".join(vocab)
+configs.max_text_length = max_len
+configs.save()
+
+# Create a data provider for the dataset
+data_provider = DataProvider(
+    dataset=dataset,
+    skip_validation=True,
+    batch_size=configs.batch_size,
+    data_preprocessors=[ImageReader(CVImage)],
+    transformers=[
+        ImageResizer(configs.width, configs.height, keep_aspect_ratio=True),
+        LabelIndexer(configs.vocab),
+        LabelPadding(max_word_length=configs.max_text_length, padding_value=len(configs.vocab)),
+        ],
+)
+
+# Split the dataset into training and validation sets
+train_data_provider, val_data_provider = data_provider.split(split = 0.9)
+
+# Augment training data with random brightness, rotation and erode/dilate
+train_data_provider.augmentors = [
+    RandomBrightness(), 
+    RandomErodeDilate(),
+    RandomSharpen(),
+    ]
+
+# Creating TensorFlow model architecture
+model = train_model(
+    input_dim = (configs.height, configs.width, 3),
+    output_dim = len(configs.vocab),
+)
+
+# Compile the model and print summary
+model.compile(
+    optimizer=tf.keras.optimizers.Adam(learning_rate=configs.learning_rate), 
+    loss=CTCloss(), 
+    metrics=[
+        CERMetric(vocabulary=configs.vocab),
+        WERMetric(vocabulary=configs.vocab)
+        ],
+    run_eagerly=False
+)
+model.summary(line_length=110)
+
+# Define callbacks
+earlystopper = EarlyStopping(monitor="val_CER", patience=20, verbose=1, mode="min")
+checkpoint = ModelCheckpoint(f"{configs.model_path}/model.h5", monitor="val_CER", verbose=1, save_best_only=True, mode="min")
+trainLogger = TrainLogger(configs.model_path)
+tb_callback = TensorBoard(f"{configs.model_path}/logs", update_freq=1)
+reduceLROnPlat = ReduceLROnPlateau(monitor="val_CER", factor=0.9, min_delta=1e-10, patience=5, verbose=1, mode="auto")
+model2onnx = Model2onnx(f"{configs.model_path}/model.h5")
+
+# Train the model
+model.fit(
+    train_data_provider,
+    validation_data=val_data_provider,
+    epochs=configs.train_epochs,
+    callbacks=[earlystopper, checkpoint, trainLogger, reduceLROnPlat, tb_callback, model2onnx],
+    workers=configs.train_workers
+)
+
+# Save training and validation datasets as csv files
+train_data_provider.to_csv(os.path.join(configs.model_path, "train.csv"))
+val_data_provider.to_csv(os.path.join(configs.model_path, "val.csv"))
