Merge pull request #568 from aws-samples/xuhan-dev

feat: update npm run build and packaging

source/infrastructure/lib/knowledge-base/knowledge-base-stack.ts

@@ -80,9 +80,7 @@ export class KnowledgeBaseStack extends NestedStack implements KnowledgeBaseStac
     });
 
     new s3Deployment.BucketDeployment(this, "DeployGlueLibs", {
-      sources: [s3Deployment.Source.asset(join(__dirname, "../../../lambda/job/dep/dist"), {
-        exclude: ["*", "!llm_bot_dep-0.1.0-py3-none-any.whl"]
-      })],
+      sources: [s3Deployment.Source.asset(join(__dirname, "../../../lambda/job/dep/dist"))],
       destinationBucket: this.glueLibS3Bucket,
     });
 

source/lambda/job/build_whl.sh

@@ -1,14 +1,28 @@
 #!/bin/bash
 
 # Change to the directory where this script is located
-# Works with both sh and bash
 SCRIPT_PATH="$0"
 echo "SCRIPT_PATH: $SCRIPT_PATH"
 SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_PATH")" && pwd)"
 cd "$SCRIPT_DIR"
 
+# Define paths using relative path for SHARED_DIR
+SHARED_DIR="$(cd "$SCRIPT_DIR/../shared" && pwd)"
+TARGET_DIR="$SCRIPT_DIR/dep/llm_bot_dep/shared"
+
+# Create target directory if it doesn't exist
+mkdir -p "$TARGET_DIR"
+
+# Copy files from shared directory to target directory
+echo "Copying shared files from $SHARED_DIR to $TARGET_DIR"
+cp -r "$SHARED_DIR"/ "$TARGET_DIR"/
+
 # Now proceed with the original commands
 cd ./dep
 pip install setuptools wheel
 
 python3 setup.py bdist_wheel
+
+# Clean up: Remove the copied files after building the wheel
+echo "Cleaning up: Removing copied files from $TARGET_DIR"
+rm -rf "$TARGET_DIR"

source/lambda/shared/langchain_integration/retrievers/databases/opensearch.py

@@ -1,19 +1,19 @@
-from langchain_core.documents import Document
+import hashlib
+import logging
+import traceback
+import uuid
+from dataclasses import dataclass
+from typing import Any, Iterable, List, Optional, Tuple, Union
+
 from langchain_community.vectorstores.opensearch_vector_search import (
-    _is_aoss_enabled,
-    _get_opensearch_client,
     _get_async_opensearch_client,
-    _import_bulk
+    _get_opensearch_client,
+    _import_bulk,
+    _is_aoss_enabled,
 )
-from typing import Optional,Iterable,List,Tuple
-import uuid 
+from langchain_core.documents import Document
 from langchain_core.pydantic_v1 import Field
-import hashlib 
-from pydantic import  Field
-from dataclasses import dataclass
-from typing import Any,Union
-import traceback
-import logging 
+from pydantic import Field
 
 logging.basicConfig(
     level=logging.INFO,
@@ -23,7 +23,7 @@
 
 
 @dataclass
-class OpenSearceBase:
+class OpenSearchBase:
     opensearch_url:Union[str,None]
     index_name:str
     client_kwargs: dict = Field(default_factory=dict)
@@ -175,7 +175,7 @@ async def asearch(self,query_dict:dict):
 
 
 
-class OpenSearchBM25Search(OpenSearceBase):
+class OpenSearchBM25Search(OpenSearchBase):
     k1: float = 1.2
     b: float = 0.75
     analyzer_type: str = "standard"
@@ -415,7 +415,7 @@ def _get_hybrid_search_index_body(
     }
 
 
-class OpenSearchHybridSearch(OpenSearceBase):
+class OpenSearchHybridSearch(OpenSearchBase):
     k1: float = 1.2
     b: float = 0.75
     analyzer_type="standard"

source/script/build.sh

@@ -33,6 +33,34 @@ if [ "$deploy_region" != "cn-north-1" ] && [ "$deploy_region" != "cn-northwest-1
     aws ecr-public get-login-password --region $ecr_login_region | docker login --username AWS --password-stdin public.ecr.aws
 fi
 
+build_lambda_asset() {
+    echo "Building Lambda Asset"
+    cd script
+    bash build-s3-dist.sh
+    cd - > /dev/null
+}
+
+build_frontend() {
+    echo "Building Frontend"
+    cd portal
+    npm install && npm run build
+    cd - > /dev/null
+}
+
+build_client_frontend() {
+    echo "Building Frontend"
+    cd cs-portal
+    npm install && npm run build
+    cd - > /dev/null
+}
+
+build_etl_package() {
+    echo "Building ETL Package"
+    cd lambda/job
+    sh build_whl.sh
+    cd - > /dev/null
+}
+
 prepare_etl_model() {
     echo "Preparing ETL Model"
     cd model/etl/code
@@ -52,34 +80,13 @@ prepare_online_model() {
     cd - > /dev/null
 }
 
-build_frontend() {
-    echo "Building Frontend"
-    cd portal
-    npm install && npm run build
-    cd - > /dev/null
-}
-
-build_client_frontend() {
-    echo "Building Frontend"
-    cd cs-portal
-    npm install && npm run build
-    cd - > /dev/null
-}
-
 build_deployment_module() {
     echo "Building Model Deployment Module"
     # curl https://aws-gcr-solutions-assets.s3.us-east-1.amazonaws.com/emd/wheels/emd-0.5.0-py3-none-any.whl -o emd-0.5.0-py3-none-any.whl && pip install emd-0.5.0-py3-none-any.whl"[all]"
     curl https://aws-gcr-solutions-assets.s3.us-east-1.amazonaws.com/emd/wheels/emd-0.6.0%2B0.6.0.mini-py3-none-any.whl -o emd-0.6.0-py3-none-any.whl && pip install emd-0.6.0-py3-none-any.whl"[all]"
     emd bootstrap
 }
 
-build_lambda_asset() {
-    echo "Building Lambda Asset"
-    cd script
-    bash build-s3-dist.sh
-    cd - > /dev/null
-}
-
 modules_prepared=""
 cd ..
 
@@ -92,9 +99,9 @@ if $ui_enabled; then
     modules_prepared="${modules_prepared}Frontend, "
 fi
 
-if $use_open_source_llm; then
-    build_deployment_module
-    modules_prepared="${modules_prepared}Model Deployment, "
+if $knowledge_base_enabled && $knowledge_base_intelliagent_enabled; then
+    build_etl_package
+    modules_prepared="${modules_prepared}ETL Package, "
 fi
 
 if $knowledge_base_enabled && $knowledge_base_intelliagent_enabled && $knowledge_base_models_enabled; then
@@ -107,6 +114,11 @@ if $knowledge_base_enabled && $knowledge_base_intelliagent_enabled && $opensearc
     modules_prepared="${modules_prepared}Online Model, "
 fi
 
+if $use_open_source_llm; then
+    build_deployment_module
+    modules_prepared="${modules_prepared}Model Deployment, "
+fi
+
 # Remove the trailing comma and space
 modules_prepared=$(echo "$modules_prepared" | sed 's/, $//')