Merge pull request #24 from TCS-2021/PredictiveAnalysis2

Predictive Analytics 2 - Implemented NER-LSTM for Claim Appeals and LSTM for Stock Prediction

Author: codit04

.gitignore

@@ -172,3 +172,4 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
+.DS_Store

Datasets/predictive-analytics2/appeals-sample-docs/Appeal_doc_1.txt

@@ -0,0 +1,5 @@
+Dear Sir/Madam,
+I am writing to appeal the denial of claim CLM654321. The reason provided was "Pre-authorization required". The treatment was provided by Dr. Alan Moore under Cigna.
+Please reconsider this decision.
+Sincerely,
+Jane Doe

Datasets/predictive-analytics2/appeals-sample-docs/Appeal_doc_2.docx

@@ -1,64 +1,43 @@
-altair==5.5.0
-astroid==3.3.9
-attrs==25.3.0
-blinker==1.9.0
-cachetools==5.5.2
-certifi==2025.1.31
-charset-normalizer==3.4.1
-click==8.1.8
-colorama==0.4.6
-dill==0.3.9
-et_xmlfile==2.0.0
-exceptiongroup==1.2.2
-gitdb==4.0.12
-GitPython==3.1.44
-idna==3.10
-iniconfig==2.1.0
-isort==6.0.1
-Jinja2==3.1.6
-joblib==1.4.2
-jsonschema==4.23.0
-jsonschema-specifications==2024.10.1
-markdown-it-py==3.0.0
-MarkupSafe==3.0.2
-mccabe==0.7.0
-mdurl==0.1.2
-narwhals==1.32.0
-numpy==2.2.4
-openpyxl==3.1.5
-packaging==24.2
-pandas==2.2.3
-patsy==1.0.1
-pillow==11.1.0
-platformdirs==4.3.7
-plotly==6.0.1
-pluggy==1.5.0
-prettytable==3.16.0
-protobuf==5.29.4
-pyarrow==19.0.1
-pydeck==0.9.1
-Pygments==2.19.1
-pylint==3.3.6
-pytest==8.3.5
-python-dateutil==2.9.0.post0
-pytz==2025.2
-referencing==0.36.2
-requests==2.32.3
-rich==13.9.4
-rpds-py==0.24.0
-scikit-learn==1.6.1
-scipy==1.15.2
-six==1.17.0
-smmap==5.0.2
-statsmodels==0.14.4
-streamlit==1.44.0
-tenacity==9.0.0
-threadpoolctl==3.6.0
-toml==0.10.2
-tomli==2.2.1
-tomlkit==0.13.2
-tornado==6.4.2
-typing_extensions==4.13.0
-tzdata==2025.2
-urllib3==2.3.0
-watchdog==6.0.0
+absl-py==2.0.0
+astunparse==1.6.3
+cachetools==5.3.0
+certifi==2022.12.7
+chardet==4.0.0
+charset-normalizer==3.1.0
+Faker==25.9.1
+flatbuffers==25.2.10
+gast==0.4.0
+google-pasta==0.2.0
+grpcio==1.71.0
+h5py==3.13.0
+idna==2.10
+ipykernel==6.25.1
+ipython==8.14.0
+Jinja2==3.1.2
+keras==3.9.2
+MarkupSafe==2.1.3
+ml-dtypes==0.4.1
+numpy==1.26.4
+opt-einsum==3.4.0
+pandas==2.0.2
+plotly==5.16.1
+protobuf==3.20.3
+pylint==3.3.6
+pytest==8.3.5
+PyPDF2==3.0.1
+python-docx==1.1.2
+requests==2.31.0
+scikit-learn==1.3.0
+six==1.16.0
+statsmodels==0.14.4
+streamlit==1.41.0
+tensorboard==2.18.0
+tensorboard-data-server==0.7.2
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.18.0
+tensorflow-io-gcs-filesystem==0.31.0
+typing_extensions==4.12.2
+urllib3==1.26.15
+Werkzeug==3.0.5
+wrapt==1.15.0
+yfinance==0.2.55

Datasets/predictive-analytics2/appeals-sample-docs/Appeal_doc_3.pdf

@@ -0,0 +1,39 @@
+"""
+data_utils.py file for Streamlit application
+"""
+
+import pandas as pd
+import yfinance as yf
+import streamlit as st
+
+def load_full_data(file):
+    """Load and preprocess historical stock data from CSV"""
+    data_frame = pd.read_csv(file, parse_dates=['Date'], dayfirst=True)
+    data_frame.columns = data_frame.columns.str.strip()
+    data_frame['Date'] = pd.to_datetime(data_frame['Date'], format='%d-%m-%Y', errors='coerce')
+    data_frame = data_frame.dropna(subset=['Date']).sort_values('Date')
+    data_frame.set_index('Date', inplace=True)
+
+    numeric_cols = ['Open', 'High', 'Low', 'Close', 'Volume']
+    data_frame[numeric_cols] = data_frame[numeric_cols].apply(pd.to_numeric, errors='coerce')
+    data_frame = data_frame.dropna(subset=numeric_cols)
+
+    return data_frame[['Stock Name'] + numeric_cols]
+
+@st.cache_data(ttl=300)
+def get_realtime_data():
+    """Fetch recent weekly NIFTY50 data"""
+    ticker_data = yf.Ticker("^NSEI")
+    return ticker_data.history(period="5y", interval="1wk")
+
+
+@st.cache_data(ttl=300)
+def get_realtime_daily_data():
+    """Fetch recent daily NIFTY50 data"""
+    ticker_data = yf.Ticker("^NSEI")
+    return ticker_data.history(period="2y", interval="1d")
+
+@st.cache_data
+def load_and_cache_file(uploaded_file):
+    """Cache uploaded CSV file content"""
+    return load_full_data(uploaded_file)

Datasets/predictive-analytics2/stocks-datasets/nifty50_5y_full_data.csv

@@ -0,0 +1 @@
+�K(.

requirements.txt

@@ -0,0 +1,65 @@
+"""
+model_utils.py file for Streamlit application
+"""
+
+import numpy as np
+from tensorflow import keras
+
+def create_sequences(data, sequence_length, feature_index):
+    """Create LSTM sequences from time series data"""
+    x_seq, y_seq = [], []
+    for i in range(len(data) - sequence_length):
+        window = data[i:i + sequence_length]
+        target = data[i + sequence_length, feature_index]
+        if not np.any(np.isnan(window)) and not np.isnan(target):
+            x_seq.append(window)
+            y_seq.append(target)
+    return np.array(x_seq), np.array(y_seq)
+
+
+def build_lstm_model(input_shape):
+    """Build a simple stacked LSTM model"""
+    keras.backend.clear_session()
+    model = keras.Sequential([
+        keras.layers.LSTM(50, return_sequences=True, input_shape=input_shape),
+        keras.layers.Dropout(0.2),
+        keras.layers.LSTM(50, return_sequences=False),
+        keras.layers.Dropout(0.2),
+        keras.layers.Dense(25),
+        keras.layers.Dense(1)
+    ])
+    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss='mean_squared_error')
+    return model
+
+
+def build_bidirectional_lstm(sequence_length):
+    """Build improved bidirectional LSTM model"""
+    model = keras.Sequential([
+        keras.layers.Bidirectional(
+            keras.layers.LSTM(128, return_sequences=True),
+            input_shape=(sequence_length, 1)),
+        keras.layers.Dropout(0.3),
+        keras.layers.Bidirectional(keras.layers.LSTM(64)),
+        keras.layers.Dropout(0.3),
+        keras.layers.Dense(32, activation='relu'),
+        keras.layers.Dense(1)
+    ])
+    model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss='mean_squared_error')
+    return model
+
+
+def rolling_forecast(model, normalized_data, sequence_length):
+    """Perform rolling forecast using trained model"""
+    predictions = []
+    if len(normalized_data) < sequence_length:
+        return predictions
+
+    last_sequence = normalized_data[:sequence_length].reshape(1, sequence_length, -1)
+
+    for i in range(sequence_length, len(normalized_data)):
+        prediction = model.predict(last_sequence, verbose=0)[0][0]
+        predictions.append(prediction)
+        next_input = normalized_data[i].reshape(1, 1, -1)
+        last_sequence = np.append(last_sequence[:, 1:, :], next_input, axis=1)
+
+    return predictions