Add cached UI values on workflows and bug fixes

.gitignore

@@ -42,6 +42,6 @@ legacy
 
 # Exe installer builders
 build/nsis/*
-!build/nsis/Intelligence Toolkit.exe
 
-app/wkhtmltox/wkhtmltox-0.12.6-1.msvc2015-win64_.exe
+app/wkhtmltox/*.exe
+/wheels

.vsts-ci.yml

@@ -7,6 +7,7 @@ trigger:
   branches:
     include:
       - main
+      - fix-st
 
 schedules:
 - cron: "0 0 * * 0"  # Runs every Sunday at midnight (UTC)
@@ -16,6 +17,7 @@ schedules:
 
 variables:
   isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
+  isTagged: startsWith(variables['Build.SourceBranch'], 'refs/tags/v')
 
 stages:
   - stage: Compliance
@@ -44,23 +46,13 @@ stages:
 
   - stage: Build
     dependsOn: []
-    condition: eq(variables.isMain, 'true')
+    condition: or(eq(variables.isTagged, 'true'), eq(variables.BUILD_EXE, 'true'))
     jobs:
       - job: build
         displayName: Build
         pool:
           vmImage: ubuntu-latest
         steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: "3.10"
-            displayName: "Use Python 3.10"
-          - task: Bash@3
-            displayName: Install Dependencies
-            inputs:
-              workingDirectory: ./
-              targetType: "inline"
-              script: pip install -r requirements.txt
           - task: Bash@3
             displayName: Build Docker Image
             inputs:
@@ -108,4 +100,45 @@ stages:
             inputs:
               workingDirectory: ./
               targetType: "inline"
-              script: docker push $(DOCKER_REGISTRY)/intel-toolkit:latest      
+              script: docker push $(DOCKER_REGISTRY)/intel-toolkit:latest   
+
+  - stage: Build_NSIS
+    dependsOn: Build
+    condition: or(eq(variables.isTagged, 'true'), eq(variables.BUILD_EXE, 'true'))
+    jobs:
+      - job: NSIS_Build
+        pool:
+          vmImage: 'windows-latest'
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: "3.10"
+            displayName: "Use Python 3.10"
+          - task: PowerShell@2
+            displayName: Install Dependencies
+            inputs:
+              targetType: "inline"
+              script: |
+                pip install -r requirements.txt
+          - task: PowerShell@2
+            displayName: 'Install NSIS'
+            inputs:
+              targetType: 'inline'
+              script: |
+                # Install NSIS using Scoop
+                iwr -useb get.scoop.sh -outfile 'install.ps1'
+                .\install.ps1 -RunAsAdmin
+                scoop update
+                scoop bucket add extras
+                scoop install nsis
+          - task: PowerShell@2
+            displayName: 'Create exe'
+            inputs:
+              targetType: 'inline'
+              script: '.\installer_script.ps1'
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: '$(System.DefaultWorkingDirectory)/build/nsis/Intelligence_toolkit_installer.exe' 
+              artifact: 'executable'  # Name of the artifact
+
+

app/Home.py

@@ -10,7 +10,7 @@ def get_transparency_faq():
         return file.read()
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="expanded", page_title='Intelligence Toolkit | Home')
+    st.set_page_config(layout="wide", initial_sidebar_state="expanded", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Home')
     load_multipage_app()
 
     transparency_faq = get_transparency_faq()

app/components/app_loader.py

@@ -17,6 +17,5 @@ def load_multipage_app():
     app_openai = ao.app_openai()
     app_openai.api_info()
 
-    #load css
-    # add_styles()
+    add_styles()
 

app/components/app_openai.py

@@ -1,12 +1,12 @@
 # Copyright (c) 2024 Microsoft Corporation. All rights reserved.
 import streamlit as st
-from util.openai_instance import get_key_env
+from util.openai_instance import get_key_env, key
 from util.SecretsHandler import SecretsHandler
 
 class app_openai:
     def _is_api_key_configured(self):
         secrets = SecretsHandler()
-        if secrets.get_secret("api_key") != '':
+        if secrets.get_secret(key) != '':
             return True
         elif get_key_env() != '':
             return True

app/components/app_terminator.py

@@ -24,7 +24,7 @@ def click():
         return click
 
     def terminate_app_btn(self):
-        if self.sv.mode.value != 'cloud':
+        if self.sv.mode.value == 'exe':
             exit_app = st.sidebar.button("🔴 Terminate application", disabled=st.session_state.off_btn_disabled, on_click=self._on_click)
             if exit_app:
                 st.text("Shutting down application...")

app/pages/Attribute_Patterns.py

@@ -4,7 +4,7 @@
 import streamlit as st
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Attribute Patterns')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Attribute Patterns')
     load_multipage_app()
     workflows.attribute_patterns.workflow.create()
 

app/pages/Data_Synthesis.py

@@ -4,7 +4,7 @@
 import streamlit as st
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Data Synthesis')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Data Synthesis')
     load_multipage_app()
     workflows.data_synthesis.workflow.create()
 

app/pages/Group_Narratives.py

@@ -4,7 +4,7 @@
 from components.app_loader import load_multipage_app
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Group Narratives')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Group Narratives')
     load_multipage_app()
     workflows.group_narratives.workflow.create()
 

app/pages/Question_Answering.py

@@ -5,7 +5,7 @@
 
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Question Answering')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Question Answering')
     load_multipage_app()
     workflows.question_answering.workflow.create()
 

app/pages/Record_Matching.py

@@ -4,7 +4,7 @@
 import streamlit as st
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Record Matching')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Record Matching')
     load_multipage_app()
     workflows.record_matching.workflow.create()
 

app/pages/Risk_Networks.py

@@ -4,7 +4,7 @@
 import streamlit as st
 
 def main():
-    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_title='Intelligence Toolkit | Risk Networks')
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Risk Networks')
     load_multipage_app()
     workflows.risk_networks.workflow.create()
 

app/pages/Settings.py

@@ -1,43 +1,42 @@
 import os
-from util.openai_instance import get_key_env
+from components.app_loader import load_multipage_app
+from util.openai_instance import get_key_env, key
 from util.SecretsHandler import SecretsHandler
 import streamlit as st
 import time
 from util.session_variables import SessionVariables
 
-key = 'openaikey'
 def on_change(handler, key = None, value = None):
     def change():
-        handler.write_secret('api_key', st.session_state[key] if key else value)
+        handler.write_secret(key, value)
     return change
 
 def main():
+    st.set_page_config(layout="wide", initial_sidebar_state="collapsed", page_icon="app/myapp.ico", page_title='Intelligence Toolkit | Settings')
+    load_multipage_app()
     st.header("Settings")
     sv = SessionVariables('home')
 
-    if key not in st.session_state:
-        st.session_state[key] = ''
-
     secrets_handler = SecretsHandler()
     placeholder = "Enter key here..."
-    secret = secrets_handler.get_secret("api_key")
-
+    secret = secrets_handler.get_secret(key)
     is_mode_cloud = sv.mode.value == 'cloud'
 
-    secret_input = st.text_input('Enter your OpenAI key', key=key, type="password", disabled=is_mode_cloud, placeholder=placeholder, value=secret, on_change=on_change(secrets_handler, key))
+    secret_input = st.text_input('Enter your OpenAI key', type="password", disabled=is_mode_cloud, placeholder=placeholder, value=secret)
 
-    if secret and len(secret):
+    if secret and len(secret) > 0:
         st.info("Your key is saved securely.")
         clear_btn = st.button("Clear local key")
 
         if clear_btn:
-            on_change(secrets_handler, value='')()
+            on_change(secrets_handler, key, value='')()
             time.sleep(0.3)
             st.rerun()
 
     if secret_input and secret_input != secret:
+        secrets_handler.write_secret(key, secret_input)
         st.rerun()
-    elif get_key_env() == '':
+    elif get_key_env() == '' and len(secret) == 0:
         st.warning("No OpenAI key found in the environment. Please insert one above.")
     elif not secret_input and not secret: 
         st.info("Using key from the environment.")

app/util/AI_API.py

@@ -63,4 +63,10 @@ def generate_text_from_message_list(messages, placeholder=None, prefix='', model
             placeholder.markdown(prefix + response, unsafe_allow_html=True)
     except Exception as e:
         print(f'Error generating from message list: {e}')
+        if '401' and 'invalid_api_key' in str(e):
+            if placeholder is not None:
+                placeholder.error(f'Error generating OpenAI response. Your key is invalid.')
+        else:
+            if placeholder is not None:
+                placeholder.error(f'Error generating OpenAI response.')
     return response

app/util/Database.py

@@ -21,8 +21,8 @@ def insert_into_embeddings(self, hash_text, embedding, username = ''):
 
 
     def insert_multiple_into_embeddings(self, embeddings, username = ""):
-        embeddings = ''.join([f"('{username}','{embedding[0]}', {embedding[1]}), " for embedding in embeddings])[:-2]
-        self.connection.execute(f"INSERT OR IGNORE INTO embeddings VALUES {embeddings}")
+        for embedding in embeddings:
+            self.connection.execute(f"INSERT OR IGNORE INTO embeddings VALUES ('{username}','{embedding[0]}', {embedding[1]})")
 
     def execute(self, query):
         return self.connection.execute(query)

app/util/Embedder.py

@@ -37,7 +37,7 @@ def encode_all(self, texts):
             if not embeddings:
                 new_texts.append((ix, text))
             else:
-                final_embeddings[ix] = np.array(embeddings)
+                final_embeddings[ix] = np.array(embeddings[0])
         print(f'Got {len(new_texts)} new texts')
         # split into batches of 2000
         pb = st.progress(0, 'Embedding text batches...')
@@ -49,12 +49,18 @@ def encode_all(self, texts):
             batch = new_texts[i:i+2000]
             batch_texts = [x[1] for x in batch]
             list_all_embeddings = []
-            embeddings = [x.embedding for x in openai.client().embeddings.create(input = batch_texts, model=self.model).data]
-            for j, (ix, text) in enumerate(batch):
-                hsh = hash(text)
-                list_all_embeddings.append((hsh, embeddings[j]))
-                final_embeddings[ix] = np.array(embeddings[j])
-            self.connection.insert_multiple_into_embeddings(list_all_embeddings) 
+            try:
+                embeddings = [x.embedding for x in openai.client().embeddings.create(input = batch_texts, model=self.model).data]
+                for j, (ix, text) in enumerate(batch):
+                    hsh = hash(text)
+                    list_all_embeddings.append((hsh, embeddings[j]))
+                    final_embeddings[ix] = np.array(embeddings[j])
+                self.connection.insert_multiple_into_embeddings(list_all_embeddings) 
+            except Exception as e:
+                if '401' and 'invalid_api_key' in str(e):
+                    raise Exception('Error generating OpenAI response. Your key is invalid.')
+                else:
+                    raise Exception('Error generating OpenAI response.')
 
         pb.empty()
         return np.array(final_embeddings)
@@ -76,9 +82,11 @@ def encode(self, text, auto_save = True):
                 if auto_save:
                     self.connection.insert_into_embeddings(hsh, embedding, self.username)
                 return embedding
-            except:
-                print(f'Error embedding text: {text}')
-                return None
+            except Exception as e:
+                if '401' and 'invalid_api_key' in str(e):
+                    raise Exception('Error generating OpenAI response. Your key is invalid.')
+                else:
+                    raise Exception('Error generating OpenAI response.')
 
 def create_embedder(cache, model=embed_model, encoder=text_encoder, max_tokens=max_embed_tokens):
     return Embedder(cache, model, encoder, max_tokens)

app/util/SecretsHandler.py

@@ -1,28 +1,27 @@
 import os
-import streamlit as st
 
 class SecretsHandler:
-    _instance = None
     _directory = ".streamlit"
+    _file_name = "app_secrets.toml"
+    _file_path = os.path.join(_directory, _file_name)
 
     def __init__(self):
         if not os.path.exists(self._directory):
             os.makedirs(self._directory)
-            with(open(os.path.join(self._directory, "secrets.toml"), "w")) as f:
+        if not os.path.exists(self._file_path):
+            with(open(self._file_path, "w+")) as f:
                 f.write("")
 
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super().__new__(cls)
-
-        return cls._instance
-
     def write_secret(self, key, value):
-        with(open(os.path.join(self._directory, "secrets.toml"), "w")) as f:
-            f.write(f"{key} = '{value}'")
+        with(open(self._file_path, "w")) as f:
+            f.write(f"{key}:{value};")
 
     def get_secret(self, key) -> str:
-        if st.secrets and key in st.secrets:
-            return st.secrets[key]
-        return ''
-
+        with(open(self._file_path, "r")) as f:
+            secrets = f.read()
+            secret_key = secrets.split(";")
+            for key_value in secret_key:
+                secret_key = key_value.split(":")[0]
+                if key == secret_key:
+                    return key_value.split(":")[1]
+        return ''

app/util/openai_instance.py

@@ -4,6 +4,7 @@
 import os
 from util.SecretsHandler import SecretsHandler
 
+key = 'openaikey'
 class _OpenAI:
     _instance = None
     _key = None
@@ -13,14 +14,14 @@ def __init__(self):
         self._secrets = SecretsHandler()
 
     def client(self):
-        if self._secrets.get_secret("api_key") != '':
-            key = st.secrets["api_key"]
+        if self._secrets.get_secret(key) != '':
+            api_key = self._secrets.get_secret(key)
         else:
-            key = get_key_env()
-        if key != self._key:
-            self._key = key
+            api_key = get_key_env()
+        if api_key != self._key:
+            self._key = api_key
             try:
-                self._instance = OpenAI(api_key=key)
+                self._instance = OpenAI(api_key=self._key)
             except Exception as e:
                 print(f'Error creating OpenAI client: {e}')