Add Request Timeouts (#1431)

* add timeout

* add iceberg exception mapping

* dont use quarkus bom, disable timeout

* nits

Author: RichardLiu2001

api/iceberg-service/build.gradle.kts

@@ -45,6 +45,8 @@ dependencies {
   implementation("com.fasterxml.jackson.core:jackson-annotations")
   implementation("com.fasterxml.jackson.core:jackson-core")
   implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.microprofile.fault.tolerance.api)
 }
 
 openApiGenerate {

api/management-service/build.gradle.kts

@@ -33,6 +33,7 @@ dependencies {
   compileOnly(libs.jakarta.annotation.api)
   compileOnly(libs.jakarta.inject.api)
   compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.microprofile.fault.tolerance.api)
   compileOnly(libs.swagger.annotations)
 
   implementation(libs.jakarta.servlet.api)

api/polaris-catalog-service/build.gradle.kts

@@ -71,6 +71,8 @@ dependencies {
   implementation("com.fasterxml.jackson.core:jackson-annotations")
   implementation("com.fasterxml.jackson.core:jackson-core")
   implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.microprofile.fault.tolerance.api)
 }
 
 openApiGenerate {

gradle/libs.versions.toml

@@ -73,6 +73,7 @@ javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0
 junit-bom = { module = "org.junit:junit-bom", version = "5.12.2" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.14.6" }
+microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.1" }
 mockito-core = { module = "org.mockito:mockito-core", version = "5.17.0" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.17.0" }
 opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.49.0" }

quarkus/defaults/src/main/resources/application.properties

@@ -86,6 +86,10 @@ quarkus.otel.sdk.disabled=true
 
 quarkus.test.integration-test-profile=it
 
+quarkus.fault-tolerance.global.timeout.enabled=false
+# quarkus.fault-tolerance.global.timeout.unit=minutes
+# quarkus.fault-tolerance.global.timeout.value=10
+
 polaris.realm-context.type=default
 polaris.realm-context.realms=POLARIS
 polaris.realm-context.header-name=Polaris-Realm

quarkus/service/build.gradle.kts

@@ -55,6 +55,7 @@ dependencies {
   implementation("io.quarkus:quarkus-opentelemetry")
   implementation("io.quarkus:quarkus-security")
   implementation("io.quarkus:quarkus-smallrye-context-propagation")
+  implementation("io.quarkus:quarkus-smallrye-fault-tolerance")
 
   implementation(libs.jakarta.enterprise.cdi.api)
   implementation(libs.jakarta.inject.api)

server-templates/api.mustache

@@ -57,6 +57,8 @@ import {{javaxPackage}}.inject.Inject;
 
 import org.apache.polaris.core.context.RealmContext;
 
+import org.eclipse.microprofile.faulttolerance.Timeout;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -108,6 +110,7 @@ public class {{classname}}  {
   @Produces({ {{#produces}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/produces}} }){{/hasProduces}}{{#hasAuthMethods}}
   {{#authMethods}}{{#isOAuth}}@RolesAllowed("**"){{/isOAuth}}{{/authMethods}}{{/hasAuthMethods}}
   @Timed("{{metricsPrefix}}.{{baseName}}.{{nickname}}")
+  @Timeout
   public Response {{nickname}}({{#isMultipart}}MultipartFormDataInput input,{{/isMultipart}}{{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{^isMultipart}}{{>formParams}},{{/isMultipart}}{{#isMultipart}}{{^isFormParam}},{{/isFormParam}}{{/isMultipart}}{{/allParams}}@Context @MeterTag(key="realm_id",expression="realmIdentifier") RealmContext realmContext,@Context SecurityContext securityContext) {
 {{! Don't log form or header params in case there are secrets, e.g., OAuth tokens }}
     LOGGER.atDebug().setMessage("Invoking {{baseName}} with params")

service/common/build.gradle.kts

@@ -90,6 +90,8 @@ dependencies {
   implementation("com.azure:azure-storage-blob")
   implementation("com.azure:azure-storage-file-datalake")
 
+  implementation(libs.microprofile.fault.tolerance.api)
+
   testImplementation(platform(libs.junit.bom))
   testImplementation("org.junit.jupiter:junit-jupiter")
   testImplementation(libs.assertj.core)

service/common/src/main/java/org/apache/polaris/service/exception/IcebergExceptionMapper.java

@@ -57,6 +57,7 @@
 import org.apache.iceberg.exceptions.ValidationException;
 import org.apache.iceberg.rest.responses.ErrorResponse;
 import org.apache.polaris.core.exceptions.FileIOUnknownHostException;
+import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.event.Level;
@@ -191,6 +192,7 @@ static int mapExceptionToResponseCode(RuntimeException rex) {
       case IllegalArgumentException e -> Status.BAD_REQUEST.getStatusCode();
       case UnsupportedOperationException e -> Status.NOT_ACCEPTABLE.getStatusCode();
       case WebApplicationException e -> e.getResponse().getStatus();
+      case TimeoutException e -> Status.REQUEST_TIMEOUT.getStatusCode();
       default -> Status.INTERNAL_SERVER_ERROR.getStatusCode();
     };
   }