Merge branch 'master' into Avoid-concurrent-issues-in-registry-during…

…-high-traffic-1
wso2 · Sep 30, 2023 · 2bfcecd · 2bfcecd
2 parents 0653b22 + 4e7119a
commit 2bfcecd
Show file tree

Hide file tree

Showing 18 changed files with 456 additions and 26 deletions.
diff --git a/modules/distribution/product/src/main/extensions/self-registration-username-request.jsp b/modules/distribution/product/src/main/extensions/self-registration-username-request.jsp
@@ -18,17 +18,25 @@
 <%@ page contentType="text/html;charset=UTF-8" language="java" %>
 
 <%@ page import="org.owasp.encoder.Encode" %>
+<%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.client.ApiException" %>
 <%@ page import="org.wso2.carbon.identity.mgt.constants.SelfRegistrationStatusCodes" %>
+<%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.client.api.ReCaptchaApi" %>
+<%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.client.model.ReCaptchaProperties" %>
 <%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.IdentityManagementEndpointConstants" %>
 <%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.IdentityManagementServiceUtil" %>
 <%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.client.model.User" %>
 <%@ page import="org.wso2.carbon.identity.mgt.endpoint.util.IdentityManagementEndpointUtil" %>
+<%@ page import="org.wso2.carbon.identity.captcha.util.CaptchaUtil" %>
+<%@ page import="java.util.Arrays" %>
+<%@ page import="java.util.HashMap" %>
 <%@ page import="java.io.File" %>
+<%@ page import="java.util.List" %>
 <%@ page import="java.util.Map" %>
 <%@ page import="java.util.Enumeration" %>
 <%@ taglib prefix="layout" uri="org.wso2.identity.apps.taglibs.layout.controller" %>
 
 <jsp:directive.include file="includes/localize.jsp"/>
+<jsp:directive.include file="tenant-resolve.jsp"/>
 <jsp:directive.include file="includes/layout-resolver.jsp"/>
 
 <%
@@ -67,6 +75,23 @@
     } else if (errorMsgObj != null) {
         errorMsg = errorMsgObj.toString();
     }
+    ReCaptchaApi reCaptchaApi = new ReCaptchaApi();
+    try {
+        ReCaptchaProperties reCaptchaProperties = reCaptchaApi.getReCaptcha(tenantDomain, true, "ReCaptcha",
+                "self-registration");
+        if (reCaptchaProperties.getReCaptchaEnabled()) {
+            Map<String, List<String>> headers = new HashMap<>();
+            headers.put("reCaptcha", Arrays.asList(String.valueOf(true)));
+            headers.put("reCaptchaAPI", Arrays.asList(reCaptchaProperties.getReCaptchaAPI()));
+            headers.put("reCaptchaKey", Arrays.asList(reCaptchaProperties.getReCaptchaKey()));
+            IdentityManagementEndpointUtil.addReCaptchaHeaders(request, headers);
+        }
+    } catch (ApiException e) {
+        request.setAttribute("error", true);
+        request.setAttribute("errorMsg", e.getMessage());
+        request.getRequestDispatcher("error.jsp").forward(request, response);
+        return;
+    }
     boolean skipSignUpEnableCheck = Boolean.parseBoolean(request.getParameter("skipsignupenablecheck"));
 %>
 
@@ -75,6 +100,15 @@
     layoutData.put("containerSize", "medium");
 %>
 
+<%
+    boolean reCaptchaEnabled = false;
+    if (request.getAttribute("reCaptcha") != null && "TRUE".equalsIgnoreCase((String) request.getAttribute("reCaptcha"))) {
+        reCaptchaEnabled = true;
+    } else if (request.getParameter("reCaptcha") != null && Boolean.parseBoolean(request.getParameter("reCaptcha"))) {
+        reCaptchaEnabled = true;
+    }
+%>
+
 <!doctype html>
 <html>
 <head>
@@ -87,6 +121,14 @@
     <% } else { %>
     <jsp:directive.include file="includes/header.jsp"/>
     <% } %>
+    <%
+        if (reCaptchaEnabled) {
+            String reCaptchaAPI = CaptchaUtil.reCaptchaAPIURL();
+    %>
+    <script src='<%=(reCaptchaAPI)%>'></script>
+    <%
+        }
+    %>
 </head>
 <body class="login-portal layout recovery-layout">
      <layout:main layoutName="<%= layout %>" layoutFileRelativePath="<%= layoutFileRelativePath %>" data="<%= layoutData %>" >
@@ -151,6 +193,23 @@
                         </div>
                         <% } %>
 
+                        <%
+                            if (reCaptchaEnabled) {
+                                String reCaptchaKey = CaptchaUtil.reCaptchaSiteKey();
+                        %>
+                        <div class="field">
+                            <div class="g-recaptcha"
+                                data-size="invisible"
+                                data-callback="onCompleted"
+                                data-action="register"
+                                data-sitekey="<%=Encode.forHtmlContent(reCaptchaKey)%>"
+                            >
+                            </div>
+                        </div>
+                        <%
+                            }
+                        %>
+
                         <div class="ui divider hidden"></div>
 
                         <div class="align-right buttons">
@@ -209,6 +268,9 @@
                 }
             }
         });
+        function onCompleted() {
+            $('#register').submit();
+         }
         function goBack() {
             window.history.back();
         }
@@ -226,6 +288,16 @@
                         console.warn("Prevented a possible double submit event");
                     } else {
                         e.preventDefault();
+                        <%
+                            if (reCaptchaEnabled) {
+                        %>
+                        if (!grecaptcha.getResponse()) {
+                            grecaptcha.execute();
+                            return;
+                        }
+                        <%
+                            }
+                        %>
                         var userName = document.getElementById("username");
                         var normalizedUsername = userName.value.trim();
                         userName.value = normalizedUsername;

diff --git a/modules/distribution/product/src/main/resources/conf/default.json b/modules/distribution/product/src/main/resources/conf/default.json
@@ -458,5 +458,6 @@
   "apim.analytics.properties.keystore_password": "$ref{keystore.primary.password}",
   "apim.analytics.properties.truststore_location": "${carbon.home}/repository/resources/security/$ref{truststore.file_name}",
   "apim.analytics.properties.truststore_password": "$ref{truststore.password}",
-  "tenant_mgt.disable_email_domain_validation": true
+  "tenant_mgt.disable_email_domain_validation": true,
+  "apim.jwt.use_kid_property": true
 }
diff --git a/modules/distribution/product/src/main/resources/conf/templates/repository/conf/broker.xml.j2 b/modules/distribution/product/src/main/resources/conf/templates/repository/conf/broker.xml.j2
@@ -101,6 +101,10 @@ This file is ciphertool compliant. Refer PRODUCT_HOME/repository/conf/security/c
             <allowSharedTopicSubscriptions>{{broker.transport.amqp.allow_shared_topic_subscriptions}}</allowSharedTopicSubscriptions>
             <allowStrictNameValidation>{{broker.transport.amqp.allow_strict_name_validation}}</allowStrictNameValidation>
 
+            <security>
+                <authorization>{{broker.transport.amqp.authorization}}</authorization>
+            </security>
+
             <!-- Refer repository/conf/advanced/qpid-config.xml for further AMQP-specific configurations.-->
         </amqp>
         <mqtt enabled="false">

diff --git a/modules/distribution/product/src/main/startup-scripts/api-manager.bat b/modules/distribution/product/src/main/startup-scripts/api-manager.bat
@@ -203,7 +203,7 @@ set CARBON_CLASSPATH=".\lib\*";%CARBON_CLASSPATH%
 if %JAVA_VERSION% GEQ 110 set CARBON_CLASSPATH=".\lib\endorsed\*";%CARBON_CLASSPATH%
 
 if %JAVA_VERSION% LEQ 18 set JAVA_VER_BASED_OPTS=-Djava.endorsed.dirs=".\lib\endorsed";"%JAVA_HOME%\jre\lib\endorsed";"%JAVA_HOME%\lib\endorsed"
-if %JAVA_VERSION% GEQ 110 set JAVA_VER_BASED_OPTS=--add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED  --add-opens=java.base/java.io=ALL-UNNAMED
+if %JAVA_VERSION% GEQ 110 set JAVA_VER_BASED_OPTS=--add-opens=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED  --add-opens=java.base/java.io=ALL-UNNAMED
 
 set CMD_LINE_ARGS=-Xbootclasspath/a:%CARBON_XBOOTCLASSPATH% -Xms256m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%CARBON_HOME%\repository\logs\heap-dump.hprof"
 set CMD_LINE_ARGS=%CMD_LINE_ARGS% -Dcom.sun.management.jmxremote -classpath %CARBON_CLASSPATH% %JAVA_OPTS% %JAVA_VER_BASED_OPTS%

diff --git a/modules/distribution/product/src/main/startup-scripts/api-manager.sh b/modules/distribution/product/src/main/startup-scripts/api-manager.sh
@@ -310,7 +310,7 @@ echo "Using Java memory options: $JVM_MEM_OPTS"
 JAVA_VER_BASED_OPTS="--add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED"
 
 if [ $java_version_formatted -ge 1700 ]; then
-    JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED"
+    JAVA_VER_BASED_OPTS="$JAVA_VER_BASED_OPTS --add-opens=java.naming/com.sun.jndi.ldap=ALL-UNNAMED --add-opens=java.base/sun.security.x509=ALL-UNNAMED"
 fi
 
 while [ "$status" = "$START_EXIT_STATUS" ]

diff --git a/modules/distribution/resources/api_templates/velocity_template.xml b/modules/distribution/resources/api_templates/velocity_template.xml
@@ -253,6 +253,7 @@ $in_sequences.get("$resource.getUriTemplate()").get($uri)
                     #set( $roleRegion = $!{endpoint_config.get("amznRoleRegion")} )
                     #set( $resourceName = $!{resource.getAmznResourceName()} )
                     #set( $resourceTimeout = $!{resource.getAmznResourceTimeout()} )
+                    #set( $isContentEncodingEnabled = $!{resource.isAmznResourceContentEncoded()} )
 <class name="org.wso2.carbon.apimgt.gateway.mediators.AWSLambdaMediator">
                     #if( $accessKey != '' )
     <property name="accessKey" value="$accessKey"/>
@@ -282,6 +283,9 @@ $in_sequences.get("$resource.getUriTemplate()").get($uri)
                     #if( $resourceTimeout != '' )
     <property name="resourceTimeout" value="$resourceTimeout"/>
                     #end
+                    #if( $isContentEncodingEnabled != '' )
+    <property name="isContentEncodingEnabled" value="$isContentEncodingEnabled"/>
+                    #end
 </class>
 <loopback />
                 ## AWS Lambda: end

diff --git a/modules/integration/tests-common/clients/publisher/src/main/resources/publisher-api.yaml b/modules/integration/tests-common/clients/publisher/src/main/resources/publisher-api.yaml
@@ -8707,6 +8707,12 @@ components:
             Name of the Authorization header used for invoking the API. If it is not set, Authorization header name specified
             in tenant or system level will be used.
           example: Authorization
+        apiKeyHeader:
+          type: string
+          pattern: '(^[^~!@#;:%^*()+={}|\\<>"'',&$\s+]*$)'
+          description: |
+            Name of the API key header used for invoking the API. If it is not set, default value `apiKey` will be used.
+          example: apiKey
         securityScheme:
           type: array
           description: |
@@ -9354,6 +9360,12 @@ components:
             Name of the Authorization header used for invoking the API. If it is not set, Authorization header name specified
             in tenant or system level will be used.
           example: Authorization
+        apiKeyHeader:
+          type: string
+          pattern: '(^[^~!@#;:%^*()+={}|\\<>"'',&$\s+]*$)'
+          description: |
+            Name of the API key header used for invoking the API. If it is not set, default value `apiKey` will be used.
+          example: apiKey
         securityScheme:
           type: array
           description: |

diff --git a/...tests-backend/src/test/java/org/wso2/am/integration/tests/header/CORSHeadersTestCase.java b/...tests-backend/src/test/java/org/wso2/am/integration/tests/header/CORSHeadersTestCase.java
@@ -83,7 +83,7 @@ public class CORSHeadersTestCase extends APIManagerLifecycleBaseTest {
     private static final String ACCESS_CONTROL_ALLOW_METHODS_HEADER_VALUE = "DELETE,POST,PUT,PATCH,GET";
     private static final String ACCESS_CONTROL_ALLOW_HEADERS_HEADER = "Access-Control-Allow-Headers";
     private static final String ACCESS_CONTROL_ALLOW_HEADERS_HEADER_VALUE
-            = "authorization,Access-Control-Allow-Origin,Content-Type,SOAPAction,Authorization";
+            = "authorization,Access-Control-Allow-Origin,Content-Type,SOAPAction,Authorization,ApiKey";
     private static final String ACCESS_CONTROL_ALLOW_CREDENTIALS_HEADER = "Access-Control-Allow-Credentials";
 
     private String accessToken;

diff --git a/...ests-backend/src/test/java/org/wso2/am/integration/tests/header/CustomHeaderTestCase.java b/...ests-backend/src/test/java/org/wso2/am/integration/tests/header/CustomHeaderTestCase.java
@@ -23,6 +23,8 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Factory;
 import org.testng.annotations.Test;
+import org.wso2.am.integration.clients.publisher.api.v1.dto.APIDTO;
+import org.wso2.am.integration.clients.store.api.v1.dto.APIKeyDTO;
 import org.wso2.am.integration.clients.store.api.v1.dto.ApplicationDTO;
 import org.wso2.am.integration.clients.store.api.v1.dto.ApplicationKeyDTO;
 import org.wso2.am.integration.clients.store.api.v1.dto.ApplicationKeyGenerateRequestDTO;
@@ -38,6 +40,7 @@
 import org.wso2.carbon.integration.common.utils.mgt.ServerConfigurationManager;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
 import static org.wso2.am.integration.tests.restapi.RESTAPITestConstants.APPLICATION_JSON_CONTENT;
 import static org.wso2.am.integration.tests.restapi.RESTAPITestConstants.AUTHORIZATION_KEY;
 
@@ -46,24 +49,23 @@
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class CustomHeaderTestCase extends APIManagerLifecycleBaseTest {
 
     private ServerConfigurationManager serverConfigurationManager;
     private final String CUSTOM_AUTHORIZATION_HEADER = "Test-Custom-Header";
+    private final String DEFAULT_API_KEY_HEADER = "ApiKey";
+    private final String CUSTOM_API_KEY_HEADER = "Custom-ApiKey-Header";
     private final String API1_NAME = "CustomAuthHeaderTestAPI1";
     private final String API1_CONTEXT = "customAuthHeaderTest1";
     private final String API1_VERSION = "1.0.0";
-    private final String APPLICATION1_NAME = "CustomHeaderTest-Application";
     private final String API_END_POINT_METHOD = "customers/123";
-
-    private final String API2_NAME = "CustomAuthHeaderTestAPI2";
-    private final String API2_CONTEXT = "customAuthHeaderTest2";
-    private final String API2_VERSION = "1.0.0";
     private String accessToken;
     private String applicationId;
     private String apiId;
+    String invocationUrl;
 
     @Factory(dataProvider = "userModeDataProvider")
     public CustomHeaderTestCase(TestUserMode userMode) {
@@ -94,22 +96,25 @@ public void setEnvironment() throws Exception {
                         APIMIntegrationConstants.APPLICATION_TIER.UNLIMITED,
                         ApplicationDTO.TokenTypeEnum.JWT);
         applicationId = applicationResponse.getData();
-    }
-
-    @Test(groups = {"wso2.am"}, description = "Set a customer Auth header for all APIs in the system. (Test ID: 3.1.1.5, 3.1.1.14)")
-    public void testSystemWideCustomAuthHeader() throws Exception {
-
-        APIIdentifier apiIdentifier1 = new APIIdentifier(user.getUserName(), API1_NAME, API1_VERSION);
 
+        // Create API
         String url = getGatewayURLHttp() + "jaxrs_basic/services/customers/customerservice";
         APIRequest apiRequest = new APIRequest(API1_NAME, API1_CONTEXT, new URL(url), new URL(url));
         apiRequest.setVersion(API1_VERSION);
         apiRequest.setProvider(user.getUserName());
         apiRequest.setTiersCollection(TIER_UNLIMITED);
-        String invocationUrl = getAPIInvocationURLHttps(API1_CONTEXT, API1_VERSION) + "/" + API_END_POINT_METHOD;
+        List<String> securitySchemes = new ArrayList<>();
+        securitySchemes.add("oauth2");
+        securitySchemes.add("api_key");
+        apiRequest.setSecurityScheme(securitySchemes);
+        invocationUrl = getAPIInvocationURLHttps(API1_CONTEXT, API1_VERSION) + "/" + API_END_POINT_METHOD;
         apiId = createPublishAndSubscribeToAPIUsingRest(apiRequest, restAPIPublisher, restAPIStore, applicationId,
                 APIMIntegrationConstants.API_TIER.UNLIMITED);
         waitForAPIDeploymentSync(user.getUserName(), API1_NAME, API1_VERSION, APIMIntegrationConstants.IS_API_EXISTS);
+    }
+
+    @Test(groups = {"wso2.am"}, description = "Set a customer Auth header for all APIs in the system. (Test ID: 3.1.1.5, 3.1.1.14)")
+    public void testSystemWideCustomAuthHeader() throws Exception {
 
         //get access token
         ArrayList<String> grantTypes = new ArrayList<>();
@@ -138,6 +143,69 @@ public void testSystemWideCustomAuthHeader() throws Exception {
                 "Response code mismatched");
     }
 
+    @Test(groups = {"wso2.am"}, description = "Invoke an API with default API Key header",
+            dependsOnMethods = "testSystemWideCustomAuthHeader")
+    public void testInvokeAPIWIthDefaultApiKeyHeader() throws Exception {
+
+        // Genarate API Keys for the application
+        APIKeyDTO apiKeyDTO = restAPIStore
+                .generateAPIKeys(applicationId, ApplicationKeyGenerateRequestDTO.KeyTypeEnum.PRODUCTION.toString(),
+                        -1, null, null);
+        assertNotNull(apiKeyDTO, "API Key generation failed");
+        String apiKey = apiKeyDTO.getApikey();
+
+        // Test whether a request can be made with the default API Key header
+        Map<String, String> requestHeaders1 = new HashMap<>();
+        requestHeaders1.put("accept", APPLICATION_JSON_CONTENT);
+        requestHeaders1.put(DEFAULT_API_KEY_HEADER, apiKey);
+        HttpResponse apiResponse1 = HttpRequestUtil.doGet(invocationUrl, requestHeaders1);
+        assertEquals(apiResponse1.getResponseCode(), Response.Status.OK.getStatusCode(),
+                "Response code mismatched");
+
+        // Test whether the 401 Unauthorized Response is returned with incorrect API Key header
+        Map<String, String> requestHeaders2 = new HashMap<>();
+        requestHeaders2.put("accept", APPLICATION_JSON_CONTENT);
+        requestHeaders2.put(CUSTOM_API_KEY_HEADER, apiKey);
+        HttpResponse apiResponse2 = HttpRequestUtil.doGet(invocationUrl, requestHeaders2);
+        assertEquals(apiResponse2.getResponseCode(), Response.Status.UNAUTHORIZED.getStatusCode(),
+                "Response code mismatched");
+    }
+
+    @Test(groups = {"wso2.am"}, description = "Invoke an API with custom API Key header",
+            dependsOnMethods = "testInvokeAPIWIthDefaultApiKeyHeader")
+    public void testInvokeAPIWIthCustomApiKeyHeader() throws Exception {
+
+        // Genarate API Keys for the application
+        APIKeyDTO apiKeyDTO = restAPIStore
+                .generateAPIKeys(applicationId, ApplicationKeyGenerateRequestDTO.KeyTypeEnum.PRODUCTION.toString(),
+                        -1, null, null);
+        assertNotNull(apiKeyDTO, "API Key generation failed");
+        String apiKey = apiKeyDTO.getApikey();
+
+        // Update the API with custom API Key header
+        APIDTO apidto = restAPIPublisher.getAPIByID(apiId);
+        apidto.setApiKeyHeader(CUSTOM_API_KEY_HEADER);
+        restAPIPublisher.updateAPI(apidto);
+        createAPIRevisionAndDeployUsingRest(apiId, restAPIPublisher);
+        Thread.sleep(10000);
+
+        // Test whether a request can be made with the custom API Key header
+        Map<String, String> requestHeaders1 = new HashMap<>();
+        requestHeaders1.put("accept", APPLICATION_JSON_CONTENT);
+        requestHeaders1.put(CUSTOM_API_KEY_HEADER, apiKey);
+        HttpResponse apiResponse1 = HttpRequestUtil.doGet(invocationUrl, requestHeaders1);
+        assertEquals(apiResponse1.getResponseCode(), Response.Status.OK.getStatusCode(),
+                "Response code mismatched");
+
+        // Test whether the 401 Unauthorized Response is returned with default API Key header
+        Map<String, String> requestHeaders2 = new HashMap<>();
+        requestHeaders2.put("accept", APPLICATION_JSON_CONTENT);
+        requestHeaders2.put(DEFAULT_API_KEY_HEADER, apiKey);
+        HttpResponse apiResponse2 = HttpRequestUtil.doGet(invocationUrl, requestHeaders2);
+        assertEquals(apiResponse2.getResponseCode(), Response.Status.UNAUTHORIZED.getStatusCode(),
+                "Response code mismatched");
+    }
+
     @AfterClass(alwaysRun = true)
     public void destroy() throws Exception {
         SubscriptionListDTO subsDTO = restAPIStore.getAllSubscriptionsOfApplication(applicationId);