Merge branch 'main' of https://github.com/microsoft/intelligence-toolkit

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    env:
+      python-version: "3.10"
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.python-version }}
+
+      - name: Install Dependencies
+        run: pip install -r requirements.txt
+
+      - name: Build Docker Image
+        run: docker build . -t intel-toolkit:latest

.vsts-ci.yml

@@ -0,0 +1,74 @@
+name: Intelligence Toolkit
+pool:
+  vmImage: ubuntu-latest
+
+trigger:
+  batch: true
+  branches:
+    include:
+      - main
+
+variables:
+  isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')]
+
+stages:
+  - stage: Compliance
+    dependsOn: []
+    jobs:
+      - job: compliance
+        displayName: Compliance
+        pool:
+          vmImage: windows-latest
+        steps:
+          - task: CredScan@3
+            inputs:
+              outputFormat: sarif
+              debugMode: false
+
+          - task: ComponentGovernanceComponentDetection@0
+            inputs:
+              scanType: "Register"
+              verbosity: "Verbose"
+              alertWarningLevel: "High"
+
+          - task: PublishSecurityAnalysisLogs@3
+            inputs:
+              ArtifactName: "CodeAnalysisLogs"
+              ArtifactType: "Container"
+
+  - stage: Build_deploy
+    dependsOn: []
+    condition: eq(variables.isMain, 'true')
+    jobs:
+      - job: build
+        displayName: Build
+        pool:
+          vmImage: ubuntu-latest
+        steps:
+          - task: UsePythonVersion@0
+            inputs:
+              versionSpec: "3.10"
+            displayName: "Use Python 3.10"
+          - task: Bash@3
+            displayName: Install Dependencies
+            inputs:
+              workingDirectory: ./
+              targetType: "inline"
+              script: pip install -r requirements.txt
+          - task: Bash@3
+            displayName: Build Docker Image
+            inputs:
+              workingDirectory: ./
+              targetType: "inline"
+              script: docker build . -t $(DOCKER_REGISTRY)/intel-toolkit:latest
+          - task: Bash@3
+            displayName: Docker Login
+            inputs:
+              targetType: "inline"
+              script: docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD) $(DOCKER_REGISTRY)
+          - task: Bash@3
+            displayName: Push Docker Image
+            inputs:
+              workingDirectory: ./
+              targetType: "inline"
+              script: docker push $(DOCKER_REGISTRY)/intel-toolkit:latest

Dockerfile

@@ -1,12 +1,10 @@
-
 # Copyright (c) Microsoft Corporation. All rights reserved.
-# Dockerfile
 # https://eng.ms/docs/more/containers-secure-supply-chain/approved-images
 FROM  mcr.microsoft.com/oryx/python:3.10
 
 RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg
 RUN apt-get update -y
-ENV IS_DOCKER True
+RUN apt-get install wkhtmltopdf -y
 
 # Install dependencies
 WORKDIR ./

README.md

@@ -1,29 +1,82 @@
-# Introduction 
+# Intelligence Toolkit
+The Intelligence Toolkit is a suite of interactive workflows for creating AI intelligence reports from real-world data sources. The toolkit is designed to help users identify patterns, answers, relationships, and risks within complex datasets, with generative AI ([OpenAI GPT models](https://platform.openai.com/docs/models/)) used to create reports on findings of interest.
 
-pip install -r requirements.txt
 
-streamlit run app/Home.py
+# Developing 
 
-## PDF export
-Install wkhtmltopdf to be able to generate the final reports in PDF:
+## Requirements
 
-Windows: [Download wkhtmltopdf installer](https://wkhtmltopdf.org/downloads.html)
+- Python 3.10 ([Download](https://www.python.org/downloads/))
 
-Linux:  `sudo apt-get install wkhtmltopdf`
+## Set up:
+1. Set up virtual environment:
 
-MacOS: `brew install homebrew/cask/wkhtmltopdf`
+    `python -m venv ./venv`
 
-# Project
+    or
+
+    `python3 -m venv ./venv`
 
-> This repo has been populated by an initial template to help get you started. Please
-> make sure to update the content to build a great experience for community-building.
+2. Install wkhtmltopdf to be able to generate the final story in PDF:
 
-As the maintainer of this project, please make a few updates:
+    Windows: [Download wkhtmltopdf installer](https://wkhtmltopdf.org/downloads.html)
 
-- Improving this README.MD file to provide a great experience
-- Updating SUPPORT.MD with content about this project's support experience
-- Understanding the security reporting process in SECURITY.MD
-- Remove this section from the README
+    Linux:  `sudo apt-get install wkhtmltopdf`
+
+    MacOS: `brew install homebrew/cask/wkhtmltopdf`
+
+## OpenAI Key
+
+### Linux:
+Open /venv/bin/activate, add the following lines at the end of the file:
+```
+    # set environment variables
+    export OPENAI_API_KEY=<OPENAI_API_KEY>
+```
+
+### Windows:
+Open venv/Scripts/Activate.ps1, add the following lines after line 167:
+```
+    $env:OPENAI_API_KEY="<OPENAI_API_KEY>"
+``` 
+
+### Running
+
+1. Run the activate: 
+
+    `source venv/bin/activate`  (Linux)
+
+    `.\venv\Scripts\Activate` (Windows with Powershell)
+
+2. Install all the dependencies with pip:
+
+    `pip install -r requirements.txt`
+
+3. Run the project using streamlit: 
+
+
+    `streamlit run app/Home.py`
+
+
+## Running with docker
+
+Download and install docker: https://www.docker.com/products/docker-desktop/
+
+Then, in the root folder, run:
+
+`docker build . -t intel-toolkit`
+
+After building, run the docker container with:
+
+`docker run -d -p 8501:8501 intel-toolkit`
+
+Open [localhost:8501](http://localhost:8501)
+
+
+# Deploying
+
+- In [this tutorial](https://dev.to/keneojiteli/deploy-a-docker-app-to-app-services-on-azure-5d3h), you can check how to create the services in azure.
+    - From there, you can deploy it manually like it's written, or use [our YAML file](/.vsts-ci.yml) to automatically deploy to your environment if you configure it. 
 
 ## Contributing
 

SUPPORT.md

@@ -1,25 +1,23 @@
-# TODO: The maintainer of this repo has not yet edited this file
+# Support
 
-**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project?
+## How to file issues and get help
 
-- **No CSS support:** Fill out this template with information about how to file issues and get help.
-- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps.
-- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide.
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates. For new issues, file your bug or
+feature request as a new Issue.
 
-*Then remove this first heading from this SUPPORT.MD file before publishing your repo.*
+For help and questions about using this project, please create a GitHub issue with your question.
 
 # Support
 
-## How to file issues and get help  
+## How to file issues and get help
 
-This project uses GitHub Issues to track bugs and feature requests. Please search the existing 
-issues before filing new issues to avoid duplicates.  For new issues, file your bug or 
+This project uses GitHub Issues to track bugs and feature requests. Please search the existing
+issues before filing new issues to avoid duplicates. For new issues, file your bug or
 feature request as a new Issue.
 
-For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 
-FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER
-CHANNEL. WHERE WILL YOU HELP PEOPLE?**.
+For help and questions about using this project, please file an issue on the repo.
 
-## Microsoft Support Policy  
+## Microsoft Support Policy
 
-Support for this **PROJECT or PRODUCT** is limited to the resources listed above.
+Support for this project is limited to the resources listed above.

app/Home.py

@@ -1,36 +1,11 @@
+# Copyright (c) 2024 Microsoft Corporation. All rights reserved.
 import streamlit as st
+from components.app_loader import load_multipage_app
 import util.mermaid as mermaid
-from streamlit_javascript import st_javascript
-import util.session_variables
-
-def get_user(sv):
-    if sv.mode.value != 'cloud':
-        return
-    css='''
-            [data-testid="stSidebarNavItems"] {
-                max-height: 100vh
-            }
-        '''
-    st.markdown(f'<style>{css}</style>', unsafe_allow_html=True)
-    js_code = """await fetch("/.auth/me")
-    .then(function(response) {return response.json();})
-    """
-    return_value = st_javascript(js_code)
-
-    username = None
-    if return_value == 0:
-        pass # this is the result before the actual value is returned 
-    elif isinstance(return_value, list) and len(return_value) > 0:
-        username = return_value[0]["user_id"]
-        sv.username.value = username
-        st.sidebar.write(f"Logged in as {username}")
-    else:
-        st.warning(f"Could not directly read username from azure active directory: {return_value}.")     
 
 def main():
     st.set_page_config(layout="wide", initial_sidebar_state="expanded", page_title='Intelligence Toolkit | Home')
-    sv = util.session_variables.SessionVariables('home')
-    get_user(sv)
+    load_multipage_app()
 
     transparency_faq = open('./app/TransparencyFAQ.md', 'r').read()
     st.markdown(transparency_faq + '\n\n' + f"""\

app/components/app_loader.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Microsoft Corporation. All rights reserved.
+from javascript.styles import add_styles
+import components.app_user as au
+import streamlit as st
+
+def load_multipage_app():
+    #Load user if logged in
+    user = au.app_user()
+    user.view_get_info()
+
+    #load css
+    add_styles()
+

app/components/app_user.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2024 Microsoft Corporation. All rights reserved.
+import streamlit as st
+from javascript.scripts import get_auth_user
+from util.session_variables import SessionVariables
+
+class app_user:
+
+    sv = None
+
+    def __init__(self, sv = None):
+        if sv is not None:
+            self.sv = sv
+        else:
+            self.sv = SessionVariables('home')
+        self.login()
+
+    def _get_info(self):
+        return self.sv.username.value 
+
+    def _set_user(self, username):
+        self.sv.username.value = username
+
+    def view_get_info(self):
+        if self.sv.username.value:
+            st.sidebar.write(f"Logged in as {self.sv.username.value}")
+
+    def _view_error_info(self, return_value):
+        st.warning(f"Could not directly read username from azure active directory: {return_value}.")     
+
+    def login(self):
+        if self.sv.mode.value != 'cloud':
+            return
+        return_value = get_auth_user()
+        username = None
+        if return_value == 0:
+            pass # this is the result before the actual value is returned 
+        elif isinstance(return_value, list) and len(return_value) > 0:
+            username = return_value[0]["user_id"]
+            self._set_user(username)
+        else:
+            self._view_error_info(return_value)

app/javascript/scripts.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2024 Microsoft Corporation. All rights reserved.
+from streamlit_javascript import st_javascript
+import streamlit as st
+
+def get_auth_user():
+    js_code = """await fetch("/.auth/me")
+    .then(function(response) {return response.json();})
+    """
+    return st_javascript(js_code)

app/javascript/styles.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2024 Microsoft Corporation. All rights reserved.
+import streamlit as st
+
+style_sidebar = '''
+    [data-testid="stSidebarNavItems"] {
+        max-height: 100vh
+    }
+'''
+
+style_pdf = '''body {
+    font-family: 'helvetica';
+}
+'''
+
+style_iframes = '''
+    iframe {
+        display: none;
+    }
+'''
+
+def add_styles():
+    st.markdown(f'''<style>
+        {style_sidebar}
+        {style_iframes}
+    </style>''', unsafe_allow_html=True)