diff --git a/infra/main.bicep b/infra/main.bicep index ee085d6..ac3f70e 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -856,6 +856,14 @@ module appConfiguration 'br/public:avm/res/app-configuration/configuration-store name: 'APP_LOGGING_LEVEL' value: 'INFO' } + { + name: 'AZURE_PACKAGE_LOGGING_LEVEL' + value: 'INFO' + } + { + name: 'AZURE_LOGGING_PACKAGES' + value: '' + } { name: 'AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME' value: '' diff --git a/infra/main_custom.bicep b/infra/main_custom.bicep index 7baa89e..f8ca201 100644 --- a/infra/main_custom.bicep +++ b/infra/main_custom.bicep @@ -859,6 +859,14 @@ module appConfiguration 'br/public:avm/res/app-configuration/configuration-store name: 'APP_LOGGING_LEVEL' value: 'INFO' } + { + name: 'AZURE_PACKAGE_LOGGING_LEVEL' + value: 'INFO' + } + { + name: 'AZURE_LOGGING_PACKAGES' + value: '' + } { name: 'AZURE_AI_AGENT_MODEL_DEPLOYMENT_NAME' value: '' diff --git a/src/backend-api/src/app/.env.example b/src/backend-api/src/app/.env.example index 877f2f6..f7444c6 100644 --- a/src/backend-api/src/app/.env.example +++ b/src/backend-api/src/app/.env.example @@ -3,3 +3,7 @@ APP_CONFIGURATION_URL="" # APP_LOGGING_ENABLE=true # APP_LOGGING_LEVEL="INFO" +# Azure Logging Configuration +# AZURE_PACKAGE_LOGGING_LEVEL="WARNING" # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL +# AZURE_LOGGING_PACKAGES="azure.core.pipeline.policies.http_logging_policy,azure.storage.blob,azure.storage.queue,azure.core,azure.identity,azure.storage,azure.core.pipeline,azure.core.pipeline.policies,azure.core.pipeline.transport,openai,openai._client,httpx,httpcore,semantic_kernel,urllib3,msal" + diff --git a/src/backend-api/src/app/libs/application/application_configuration.py b/src/backend-api/src/app/libs/application/application_configuration.py index 94a3686..bb8d9b8 100644 --- a/src/backend-api/src/app/libs/application/application_configuration.py +++ b/src/backend-api/src/app/libs/application/application_configuration.py @@ -33,6 +33,10 @@ class Configuration(_configuration_base, KernelBaseSettings): app_logging_level: str = Field(default="INFO") app_sample_variable: str = Field(default="Hello World!") + # Azure logging configuration + azure_package_logging_level: str = Field(default="WARNING", alias="AZURE_PACKAGE_LOGGING_LEVEL") + azure_logging_packages: str | None = Field(default=None, alias="AZURE_LOGGING_PACKAGES") + global_llm_service: str | None = "AzureOpenAI" cosmos_db_process_log_container: str | None = Field( default=None, env="COSMOS_DB_PROCESS_LOG_CONTAINER" diff --git a/src/backend-api/src/app/libs/base/application_base.py b/src/backend-api/src/app/libs/base/application_base.py index b097742..7b265c0 100644 --- a/src/backend-api/src/app/libs/base/application_base.py +++ b/src/backend-api/src/app/libs/base/application_base.py @@ -60,6 +60,12 @@ def __init__(self, env_file_path: str | None = None, **data): ) logging.basicConfig(level=logging_level) + # Configure Azure package logging levels only if packages are specified + if self.application_context.configuration.azure_logging_packages: + azure_level = getattr(logging, self.application_context.configuration.azure_package_logging_level.upper(), logging.WARNING) + for logger_name in filter(None, (pkg.strip() for pkg in self.application_context.configuration.azure_logging_packages.split(','))): + logging.getLogger(logger_name).setLevel(azure_level) + # Initialize the application self.initialize() diff --git a/src/backend-api/src/app/routers/router_debug.py b/src/backend-api/src/app/routers/router_debug.py index 7e9f32b..1622a5d 100644 --- a/src/backend-api/src/app/routers/router_debug.py +++ b/src/backend-api/src/app/routers/router_debug.py @@ -19,6 +19,8 @@ async def get_config_debug(request: Request): config_dict = { "app_logging_enable": config.app_logging_enable, "app_logging_level": config.app_logging_level, + "azure_package_logging_level": config.azure_package_logging_level, + "azure_logging_packages": config.azure_logging_packages, "cosmos_db_account_url": config.cosmos_db_account_url, "cosmos_db_database_name": config.cosmos_db_database_name, "cosmos_db_process_container": config.cosmos_db_process_container, diff --git a/src/backend-api/src/tests/application/test_app_configuration.py b/src/backend-api/src/tests/application/test_app_configuration.py index df7bb5a..c03d2a0 100644 --- a/src/backend-api/src/tests/application/test_app_configuration.py +++ b/src/backend-api/src/tests/application/test_app_configuration.py @@ -28,6 +28,14 @@ def test_configuration_fields(): assert isinstance(config.app_logging_enable, bool) assert config.app_logging_enable is True + assert hasattr(config, "azure_package_logging_level") + assert isinstance(config.azure_package_logging_level, str) + assert config.azure_package_logging_level == "INFO" + + assert hasattr(config, "azure_logging_packages") + assert isinstance(config.azure_logging_packages, list) + assert config.azure_logging_packages == [] + assert hasattr(config, "app_logging_level") assert isinstance(config.app_logging_level, str) assert config.app_logging_level == "INFO" diff --git a/src/backend-api/test_config.py b/src/backend-api/test_config.py index eec2ff8..9aca087 100644 --- a/src/backend-api/test_config.py +++ b/src/backend-api/test_config.py @@ -20,6 +20,8 @@ def test_configuration(): config_values = { "app_logging_enable": config.app_logging_enable, "app_logging_level": config.app_logging_level, + "azure_package_logging_level": config.azure_package_logging_level, + "azure_logging_packages": config.azure_logging_packages, "cosmos_db_account_url": config.cosmos_db_account_url, "cosmos_db_database_name": config.cosmos_db_database_name, "cosmos_db_process_container": config.cosmos_db_process_container, diff --git a/src/processor/.env.example b/src/processor/.env.example new file mode 100644 index 0000000..d083b13 --- /dev/null +++ b/src/processor/.env.example @@ -0,0 +1,19 @@ +APP_CONFIGURATION_URL="" + +# Application Logging Configuration +# APP_LOGGING_ENABLE=true +# APP_LOGGING_LEVEL="INFO" + +# Azure Logging Configuration +# AZURE_PACKAGE_LOGGING_LEVEL="WARNING" # Options: DEBUG, INFO, WARNING, ERROR, CRITICAL +# AZURE_LOGGING_PACKAGES="azure.core.pipeline.policies.http_logging_policy,azure.storage.blob,azure.storage.queue,azure.core,azure.identity,azure.storage,azure.core.pipeline,azure.core.pipeline.policies,azure.core.pipeline.transport,openai,openai._client,httpx,httpcore,semantic_kernel,urllib3,msal" + +# Database Configuration +# COSMOS_DB_ACCOUNT_URL="http://" +# COSMOS_DB_DATABASE_NAME="" +# COSMOS_DB_CONTAINER_NAME="" + +# Storage Configuration +# STORAGE_QUEUE_ACCOUNT="http://" +# STORAGE_ACCOUNT_PROCESS_QUEUE="http://" +# STORAGE_QUEUE_NAME="processes-queue" \ No newline at end of file diff --git a/src/processor/src/libs/application/application_configuration.py b/src/processor/src/libs/application/application_configuration.py index 1719b2c..eb7f159 100644 --- a/src/processor/src/libs/application/application_configuration.py +++ b/src/processor/src/libs/application/application_configuration.py @@ -29,6 +29,10 @@ class Configuration(_configuration_base): # api_key: str app_logging_enable: bool = Field(default=False, alias="APP_LOGGING_ENABLE") app_logging_level: str = Field(default="INFO", alias="APP_LOGGING_LEVEL") + + # Azure logging configuration + azure_package_logging_level: str = Field(default="WARNING", alias="AZURE_PACKAGE_LOGGING_LEVEL") + azure_logging_packages: str | None = Field(default=None, alias="AZURE_LOGGING_PACKAGES") cosmos_db_account_url: str = Field( default="http://", alias="COSMOS_DB_ACCOUNT_URL" ) diff --git a/src/processor/src/libs/base/ApplicationBase.py b/src/processor/src/libs/base/ApplicationBase.py index 0f6b178..0753ff2 100644 --- a/src/processor/src/libs/base/ApplicationBase.py +++ b/src/processor/src/libs/base/ApplicationBase.py @@ -77,6 +77,12 @@ async def initialize_async(self): # Ensure non-debug mode suppresses all debug messages logging.basicConfig(level=logging.WARNING) + # Configure Azure package logging levels only if packages are specified + if self.app_context.configuration.azure_logging_packages: + azure_level = getattr(logging, self.app_context.configuration.azure_package_logging_level.upper(), logging.WARNING) + for logger_name in filter(None, (pkg.strip() for pkg in self.app_context.configuration.azure_logging_packages.split(','))): + logging.getLogger(logger_name).setLevel(azure_level) + # Always suppress semantic kernel debug messages unless explicitly in debug mode if not self.debug_mode: logging.getLogger("semantic_kernel").setLevel(logging.WARNING) diff --git a/src/processor/src/utils/logging_utils.py b/src/processor/src/utils/logging_utils.py index 58448a3..3c083c2 100644 --- a/src/processor/src/utils/logging_utils.py +++ b/src/processor/src/utils/logging_utils.py @@ -18,7 +18,7 @@ from semantic_kernel.exceptions import ServiceException -def configure_application_logging(debug_mode: bool = False): +def configure_application_logging(debug_mode: bool = False, config=None): """ Comprehensive logging configuration with third-party suppression. @@ -26,13 +26,18 @@ def configure_application_logging(debug_mode: bool = False): Args: debug_mode: If True, allows some debug logging. If False, suppresses all debug output. + config: Configuration object with logging settings. If None, uses default settings. """ # Set root logger level if debug_mode: logging.basicConfig(level=logging.DEBUG) print("🐛 Debug logging enabled") else: - logging.basicConfig(level=logging.INFO) + # Use configuration if available, otherwise default to INFO + basic_level = logging.INFO + if config and hasattr(config, 'app_logging_level'): + basic_level = getattr(logging, config.app_logging_level.upper(), logging.INFO) + logging.basicConfig(level=basic_level) # Comprehensive list of verbose loggers to suppress verbose_loggers = [ @@ -83,33 +88,43 @@ def configure_application_logging(debug_mode: bool = False): "charset_normalizer", ] - # Set levels for all verbose loggers - for logger_name in verbose_loggers: - logger = logging.getLogger(logger_name) - if debug_mode: - # In debug mode, still reduce verbosity to INFO for most, WARNING for HTTP - if any( - http_term in logger_name.lower() - for http_term in ["http", "client", "connection", "pipeline"] - ): - logger.setLevel(logging.WARNING) + # Configure Azure package logging levels from configuration only if packages are specified + if config and hasattr(config, 'azure_logging_packages') and hasattr(config, 'azure_package_logging_level') and config.azure_logging_packages: + # Use configuration-based approach + azure_level = getattr(logging, config.azure_package_logging_level.upper(), logging.WARNING) + package_list = [pkg.strip() for pkg in config.azure_logging_packages.split(',')] + for logger_name in package_list: + if logger_name: # Skip empty strings + logging.getLogger(logger_name).setLevel(azure_level) + else: + # Fallback to existing logic for backward compatibility + # Set levels for all verbose loggers + for logger_name in verbose_loggers: + logger = logging.getLogger(logger_name) + if debug_mode: + # In debug mode, still reduce verbosity to INFO for most, WARNING for HTTP + if any( + http_term in logger_name.lower() + for http_term in ["http", "client", "connection", "pipeline"] + ): + logger.setLevel(logging.WARNING) + else: + logger.setLevel(logging.INFO) else: - logger.setLevel(logging.INFO) - else: - # In production, suppress to WARNING for all - logger.setLevel(logging.WARNING) - - # Special cases: These are ALWAYS set to WARNING due to extreme verbosity - always_warning_loggers = [ - "azure.core.pipeline.policies.http_logging_policy", - "httpx", - "httpcore", - "openai._client", - "urllib3.connectionpool", - ] + # In production, suppress to WARNING for all + logger.setLevel(logging.WARNING) - for logger_name in always_warning_loggers: - logging.getLogger(logger_name).setLevel(logging.WARNING) + # Special cases: These are ALWAYS set to WARNING due to extreme verbosity + always_warning_loggers = [ + "azure.core.pipeline.policies.http_logging_policy", + "httpx", + "httpcore", + "openai._client", + "urllib3.connectionpool", + ] + + for logger_name in always_warning_loggers: + logging.getLogger(logger_name).setLevel(logging.WARNING) # Set environment variables to suppress verbose output at the source os.environ.setdefault("HTTPX_LOG_LEVEL", "WARNING")