Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion ui-backend/catalog-ui-search/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,11 @@
<artifactId>catalog-rest-api</artifactId>
<version>${ddf.version}</version>
</dependency>
<dependency>
<groupId>ddf.catalog.rest</groupId>
<artifactId>catalog-rest-endpoint</artifactId>
<version>${ddf.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down Expand Up @@ -492,7 +497,8 @@
ows-v_1_1_0-schema,
jaxb2-basics-runtime,
catalog-rest-service,
owasp-java-html-sanitizer
owasp-java-html-sanitizer,
catalog-rest-endpoint
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Usually it is bad practice to embed bundles. I would expect that the rest endpoint classes you are trying to access would be available within the OSGI service registry and so embedding this jar shouldn't be necessary

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, we should not be embedding this bundle here as it will likely cause unintended consequences. Also see my other comments on its use later as this shouldn't be needed.

</Embed-Dependency>
<Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
<Import-Package>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,23 @@
import javax.servlet.MultipartConfigElement;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriBuilder;
import javax.ws.rs.core.UriInfo;
import org.apache.commons.codec.CharEncoding;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.cxf.jaxrs.impl.UriBuilderImpl;
import org.apache.http.HttpStatus;
import org.codice.ddf.catalog.ui.config.ConfigurationApplication;
import org.codice.ddf.catalog.ui.util.jaxrs.JaxRsHttpHeaders;
import org.codice.ddf.catalog.ui.util.jaxrs.JaxRsUriInfo;
import org.codice.ddf.catalog.ui.util.multipart.CleanableMultipartBody;
import org.codice.ddf.catalog.ui.util.multipart.CleanableMultipartBodyFactory;
import org.codice.ddf.endpoints.rest.RESTEndpoint;
import org.codice.ddf.rest.api.CatalogService;
import org.codice.ddf.rest.api.CatalogServiceException;
import org.slf4j.Logger;
Expand All @@ -66,16 +74,27 @@ public class CatalogApplication implements SparkApplication {

private static final String HEADER_ACCEPT_RANGES = "Accept-Ranges";

private static final String HEADER_ID = "id";

private static final String BYTES = "bytes";

private static final String CATALOG_PATH = "/catalog/";

private static final String CATALOG_ID_PATH = "/catalog/:id";

private static final String TRANSFORM = "transform";

private final ConfigurationApplication config;

private CatalogService catalogService;

public CatalogApplication(CatalogService catalogService) {
private RESTEndpoint restEndpoint;

public CatalogApplication(
CatalogService catalogService, RESTEndpoint restEndpoint, ConfigurationApplication config) {
this.catalogService = catalogService;
this.restEndpoint = restEndpoint;
this.config = config;
}

@Override
Expand Down Expand Up @@ -141,35 +160,52 @@ public void init() {
});

post(
"/catalog/",
CATALOG_PATH,
(req, res) -> {
if (req.contentType().startsWith("multipart/")) {
req.attribute(
ECLIPSE_MULTIPART_CONFIG,
new MultipartConfigElement(System.getProperty(JAVA_IO_TMPDIR)));
try {
LOGGER.debug("POST Path: {}", CATALOG_PATH);
String contentType = req.contentType();

// Convert req and res for RESTEndpoint
HttpServletRequest httpRequest = req.raw();
HttpHeaders headers = new JaxRsHttpHeaders(req);
UriInfo uriInfo = new JaxRsUriInfo(req);
String transformerParam = req.queryParams(TRANSFORM);
InputStream inputStream = httpRequest.getInputStream();

if (contentType.startsWith("multipart/")) {
LOGGER.debug("POST Path: {} multipart", CATALOG_PATH);
CleanableMultipartBody multipartBody =
CleanableMultipartBodyFactory.create(
httpRequest, config.getMaximumUploadSize(), config.getMaxFileSizeInMemory());

javax.ws.rs.core.Response response =
restEndpoint.addDocument(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❓ I don't understand the reasoning for passing this off to the DDF RESTEndpoint. The DDF RESTEndpoint does basically the same thing that the original addDocument method in this class did by calling the catalogService.addDocument method. It seems to me that the issue here is related to the original code loading in the whole file with the new ByteArrayInputStream(req.bodyAsBytes())) call. You should either use a TemporaryFileBackedInputStream here or get the input stream from the original request. Then use the appropriate method(s) in the catalog service and close any streams here afterwards.

In the end, though, there is no good reason to embed and use the RESTEndpoint code here that is a simple wrapper around the catalog service calls that we can already make here in this class. At a minimum, we should change this class' addDocument method to behave like the one in the RESTEndpoint rather than redirecting through that endpoint.

Copy link
Author

@e-hosea e-hosea Sep 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried your suggestions..

  1. With Input Stream
    Code:
post(
        "/catalog/",
        (req, res) -> {
          if (req.contentType().startsWith("multipart/")) {
            long MAX_UPLOAD_SIZE = 5L * 1024 * 1024 * 1024;
            int FILE_SIZE_THRESHOLD = 50 * 1024 * 1024;
            req.attribute(
                ECLIPSE_MULTIPART_CONFIG,
                new MultipartConfigElement(
                    System.getProperty(JAVA_IO_TMPDIR),
                    MAX_UPLOAD_SIZE,
                    MAX_UPLOAD_SIZE,
                    FILE_SIZE_THRESHOLD));

            try (InputStream inputStream = req.raw().getInputStream()) {

              return addDocument(
                  res,
                  req.raw().getRequestURL(),
                  req.contentType(),
                  req.queryParams(TRANSFORM),
                  req.raw(),
                  inputStream);

            } catch (IOException e) {
              res.status(500); // Set HTTP response status for errors
              return "Error processing input stream: " + e.getMessage();
            }
          }

Error:

2025-09-08T06:48:15,324Z | INFO  | tp1436683846-360 | AbstractCatalogService           | t.service.AbstractCatalogService  955 | 628 - catalog-rest-service-impl - 2.29.21 | Error while storing entry in catalog: 
ddf.catalog.source.IngestException: Could not create metacard.
	at ddf.catalog.impl.operations.OperationsMetacardSupport.generateMetacardAndContentItems(OperationsMetacardSupport.java:177) ~[?:?]
	at ddf.catalog.impl.operations.CreateOperations.create(CreateOperations.java:157) ~[?:?]
	at ddf.catalog.impl.CatalogFrameworkImpl.create(CatalogFrameworkImpl.java:237) ~[?:?]
	at Proxy28cb377a_2e8c_4bf3_afe8_5c6cd8757576.create(Unknown Source) ~[?:?]
	at org.codice.ddf.rest.service.AbstractCatalogService.addDocument(AbstractCatalogService.java:930) ~[?:?]
	at org.codice.ddf.rest.service.AbstractCatalogService.addDocument(AbstractCatalogService.java:893) ~[?:?]
	at Proxyd9efd43f_b5c7_4677_9eee_a90775b3fcfd.addDocument(Unknown Source) ~[?:?]
	at org.codice.ddf.catalog.ui.catalog.CatalogApplication.addDocument(CatalogApplication.java:412) ~[?:?]
	at org.codice.ddf.catalog.ui.catalog.CatalogApplication.lambda$init$7(CatalogApplication.java:171) ~[?:?]
	at spark.RouteImpl$1.handle(RouteImpl.java:72) ~[?:?]
	at spark.http.matching.Routes.execute(Routes.java:61) ~[?:?]
	at spark.http.matching.MatcherFilter.doFilter(MatcherFilter.java:134) ~[?:?]
	at org.codice.ddf.catalog.ui.SparkServlet.service(SparkServlet.java:151) ~[?:?]
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:584) ~[bundleFile:4.0.4]
	at org.ops4j.pax.web.service.spi.servlet.OsgiInitializedServlet.service(OsgiInitializedServlet.java:102) ~[bundleFile:?]
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1656) ~[bundleFile:9.4.57.v20241219]
	at org.ops4j.pax.web.service.spi.servlet.OsgiFilterChain.doFilter(OsgiFilterChain.java:113) ~[bundleFile:?]
	at org.ops4j.pax.web.service.jetty.internal.PaxWebServletHandler.doHandle(PaxWebServletHandler.java:334) ~[bundleFile:?]
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:560) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:505) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:234) ~[bundleFile:9.4.57.v20241219]
	at org.ops4j.pax.web.service.jetty.internal.PrioritizedHandlerCollection.handle(PrioritizedHandlerCollection.java:96) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:67) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.DoPrivilegedFilter.lambda$doFilter$0(DoPrivilegedFilter.java:36) ~[bundleFile:?]
	at java.security.AccessController.doPrivileged(AccessController.java:571) ~[?:?]
	at org.codice.ddf.pax.web.jetty.DoPrivilegedFilter.doFilter(DoPrivilegedFilter.java:33) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at com.connexta.isri.security.authorizedroles.impl.AuthorizedUserRoleHttpFilter.doFilter(AuthorizedUserRoleHttpFilter.java:42) ~[?:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at com.connexta.isri.exercisemode.ExerciseModeHttpFilter.doFilter(ExerciseModeHttpFilter.java:38) ~[?:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ResponseFilter.doFilter(ResponseFilter.java:89) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.TraceContextFilter.doFilter(TraceContextFilter.java:54) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ClientInfoFilter.doFilter(ClientInfoFilter.java:59) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at org.codice.ddf.metrics.servlet.ServletMetrics.doFilter(ServletMetrics.java:44) ~[?:?]
	at org.codice.ddf.pax.web.jetty.ProxyHttpFilterChain.doFilter(ProxyHttpFilterChain.java:65) ~[bundleFile:?]
	at org.codice.ddf.pax.web.jetty.DelegatingHttpFilterHandler.handle(DelegatingHttpFilterHandler.java:101) ~[bundleFile:?]
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:722) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.Server.handle(Server.java:516) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) ~[bundleFile:9.4.57.v20241219]
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) ~[bundleFile:9.4.57.v20241219]
	at java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: ddf.catalog.source.IngestException: Could not copy bytes of content message.
	at ddf.catalog.impl.operations.OperationsMetacardSupport.generateMetacardAndContentItems(OperationsMetacardSupport.java:136) ~[?:?]
	... 73 more
Caused by: java.io.IOException: Stream closed
	at java.io.BufferedInputStream.getInIfOpen(BufferedInputStream.java:169) ~[?:?]
	at java.io.BufferedInputStream.implTransferTo(BufferedInputStream.java:652) ~[?:?]
	at java.io.BufferedInputStream.transferTo(BufferedInputStream.java:631) ~[?:?]
	at java.nio.file.Files.copy(Files.java:3151) ~[?:?]
	at ddf.catalog.impl.operations.OperationsMetacardSupport.generateMetacardAndContentItems(OperationsMetacardSupport.java:118) ~[?:?]
	... 73 more
  1. With TemporaryFileBackedInputStream - I can't find this in google

  2. Modifying addDocument() in CatalogApplication - will send the details through email

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy and pasting from email

  1. Modifying addDocument() in CatalogApplication
     
    Code:
post(
        "/catalog/",
        (req, res) -> {
          if (req.contentType().startsWith("multipart/")) {
            long MAX_UPLOAD_SIZE = 5L * 1024 * 1024 * 1024;
            int FILE_SIZE_THRESHOLD = 50 * 1024 * 1024;
            req.attribute(
                ECLIPSE_MULTIPART_CONFIG,
                new MultipartConfigElement(
                    System.getProperty(JAVA_IO_TMPDIR),
                    MAX_UPLOAD_SIZE,
                    MAX_UPLOAD_SIZE,
                    FILE_SIZE_THRESHOLD));
 
            try (CloseableMultipartBody multipartBody =
                CloseableMultipartBodyFactory.create(
                    req.raw(), MAX_UPLOAD_SIZE, FILE_SIZE_THRESHOLD)) {
 
              return addDocument(
                  res,
                  req.raw().getRequestURL(),
                  req.contentType(),
                  req.queryParams(TRANSFORM),
                  multipartBody,
                  req.raw().getInputStream());
            }

 

I modified addDocument() to accept MultipartBody instead of HttpServletRequest

private String addDocument(
      Response res,
      StringBuffer requestUrl,
      String contentType,
      String transformerParam,
      MultipartBody multipartBody,
      //      HttpServletRequest httpServletRequest,
      InputStream inputStream) {
    try {
      List<String> contentTypeList = ImmutableList.of(contentType);
      String id =
          catalogService.addDocument(contentTypeList, multipartBody, transformerParam, inputStream);
 
      URI uri = new URI(requestUrl.toString());
      UriBuilder uriBuilder = new UriBuilderImpl(uri).path("/" + id);
 
      res.status(HttpStatus.SC_CREATED);
      res.header("Location", uriBuilder.build().toString());
      res.header(Metacard.ID, id);
      return "";
 
    } catch (CatalogServiceException | URISyntaxException e) {
      return createBadRequestResponse(res, e.getMessage());
    }
  }
 

Error:
2025-09-08T06:29:35,773Z | WARN  | p1085609986-1836 | ContentTransferHandler           | ttransfer.ContentTransferHandler   95 | 682 - content-transfer-handler - 2.10.0.0-SNAPSHOT | Failed to process replication content transfer operation: transfer
java.lang.IllegalStateException: java.lang.OutOfMemoryError: Required array length 2147483639 + 5798 is too large
        at org.eclipse.jetty.http.MultiPartFormInputStream.throwIfError(MultiPartFormInputStream.java:517) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartFormInputStream.getParts(MultiPartFormInputStream.java:470) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.server.MultiParts$MultiPartsHttpParser.getParts(MultiParts.java:74) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.server.Request.getParts(Request.java:2450) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.server.Request.getParts(Request.java:2420) ~[bundleFile:9.4.57.v20241219]
        at javax.servlet.http.HttpServletRequestWrapper.getParts(HttpServletRequestWrapper.java:317) ~[bundleFile:4.0.4]
        at javax.servlet.http.HttpServletRequestWrapper.getParts(HttpServletRequestWrapper.java:317) ~[bundleFile:4.0.4]
        at io.javalin.http.util.MultipartUtil.getUploadedFiles(MultipartUtil.kt:23) ~[?:?]
        at io.javalin.http.Context.uploadedFiles(Context.kt:158) ~[?:?]
        at io.javalin.http.Context.uploadedFile(Context.kt:154) ~[?:?]
        at com.connexta.isri.replication.ContentTransferContext.getUploadedFile(ContentTransferContext.java:28) ~[?:?]
        at com.connexta.isri.replication.contenttransfer.ContentTransferHandler.handleTransfer(ContentTransferHandler.java:100) ~[?:?]
        at com.connexta.isri.replication.contenttransfer.ContentTransferHandler.handle(ContentTransferHandler.java:71) ~[?:?]
        at Proxyb4659e8d_67ed_4dae_a2e8_331899a99267.handle(Unknown Source) ~[?:?]
       …
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) ~[bundleFile:9.4.57.v20241219]
        at java.lang.Thread.run(Thread.java:1583) [?:?]
Caused by: java.lang.OutOfMemoryError: Required array length 2147483639 + 5798 is too large
        at jdk.internal.util.ArraysSupport.hugeLength(ArraysSupport.java:752) ~[?:?]
        at jdk.internal.util.ArraysSupport.newLength(ArraysSupport.java:745) ~[?:?]
        at java.io.ByteArrayOutputStream.ensureCapacity(ByteArrayOutputStream.java:100) ~[?:?]
        at java.io.ByteArrayOutputStream.write(ByteArrayOutputStream.java:132) ~[?:?]
        at org.eclipse.jetty.http.MultiPartFormInputStream$MultiPart.write(MultiPartFormInputStream.java:178) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartFormInputStream$Handler.content(MultiPartFormInputStream.java:741) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartParser.parseOctetContent(MultiPartParser.java:645) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartParser.parse(MultiPartParser.java:214) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartFormInputStream.parse(MultiPartFormInputStream.java:587) ~[bundleFile:9.4.57.v20241219]
        at org.eclipse.jetty.http.MultiPartFormInputStream.getParts(MultiPartFormInputStream.java:469) ~[bundleFile:9.4.57.v20241219]
        ... 84 more
Suppressed: java.lang.Throwable at org.eclipse.jetty.http.MultiPartFormInputStream.throwIfError(MultiPartFormInputStream.java:512) ~[bundleFile:9.4.57.v20241219] at org.eclipse.jetty.http.MultiPartFormInputStream.getParts(MultiPartFormInputStream.java:470) ~[bundleFile:9.4.57.v20241219] at org.eclipse.jetty.server.MultiParts$MultiPartsHttpParser.getParts(MultiParts.java:74) ~[bundleFile:9.4.57.v20241219] at org.eclipse.jetty.server.Request.getParts(Request.java:2450) ~[bundleFile:9.4.57.v20241219] at org.eclipse.jetty.server.Request.getParts(Request.java:2420) ~[bundleFile:9.4.57.v20241219] at
…org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) ~[bundleFile:9.4.57.v20241219] at java.lang.Thread.run(Thread.java:1583) [?:?]
 

Even though /apps/ddf/etc/org.codice.ddf.catalog.ui.config has been modified and ddf restarted
maxFileSizeInMemory=I"50000000"
maximumUploadSize=L"10485760000"
 
But even with the error in Content Transfer Handler, the product was uploaded successfully, searchable and downloadable.

headers, uriInfo, httpRequest, multipartBody, transformerParam, inputStream);

multipartBody.cleanup();

return setResponse(
res, httpRequest.getRequestURL(), response.getHeaderString(HEADER_ID));
}

return addDocument(
res,
req.raw().getRequestURL(),
req.contentType(),
req.queryParams(TRANSFORM),
req.raw(),
new ByteArrayInputStream(req.bodyAsBytes()));
}
if (contentType.startsWith("text/") || contentType.startsWith("application/")) {
LOGGER.debug("POST Path: {} text/application", CATALOG_PATH);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: slightly misleading debug statement. The string "text/application" in this context looks like a MIME type, but it's not. I get what you mean, but it confuse someone looking that the logs. Suggestion: "text/* or application/*"

javax.ws.rs.core.Response response =
restEndpoint.addDocument(
headers, uriInfo, httpRequest, transformerParam, inputStream);

if (req.contentType().startsWith("text/")
|| req.contentType().startsWith("application/")) {
return addDocument(
res,
req.raw().getRequestURL(),
req.contentType(),
req.queryParams(TRANSFORM),
null,
new ByteArrayInputStream(req.bodyAsBytes()));
}
return setResponse(
res, httpRequest.getRequestURL(), response.getHeaderString(HEADER_ID));
}

res.status(HttpStatus.SC_NOT_FOUND);
return res;
res.status(HttpStatus.SC_NOT_FOUND);
return "Not Found";
} catch (Exception e) {
LOGGER.error("Unexpected error in request handler", e);
res.status(HttpStatus.SC_INTERNAL_SERVER_ERROR);
return "Internal Server Error";
}
});

put(
Expand Down Expand Up @@ -373,19 +409,8 @@ private String updateDocument(
}
}

private String addDocument(
Response res,
StringBuffer requestUrl,
String contentType,
String transformerParam,
HttpServletRequest httpServletRequest,
InputStream inputStream) {
private String setResponse(Response res, StringBuffer requestUrl, String id) {
try {
List<String> contentTypeList = ImmutableList.of(contentType);
String id =
catalogService.addDocument(
contentTypeList, httpServletRequest, transformerParam, inputStream);

URI uri = new URI(requestUrl.toString());
UriBuilder uriBuilder = new UriBuilderImpl(uri).path("/" + id);

Expand All @@ -394,7 +419,7 @@ private String addDocument(
res.header(Metacard.ID, id);
return "";

} catch (CatalogServiceException | URISyntaxException e) {
} catch (URISyntaxException e) {
return createBadRequestResponse(res, e.getMessage());
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,8 @@ public class ConfigurationApplication implements SparkApplication {

private String mapHome = "";

private int maximumUploadSize = 1_048_576;
private long maximumUploadSize = 1_048_576;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

✏️ delete old int variable and update ddf-ui/ui-backend/catalog-ui-search/src/main/java/org/codice/ddf/catalog/ui/config/ConfigurationApplication.java to Long

Will want to add maxFileSizeInMemory to the above file too

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmmm I'm confused. It's already been done like that.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry - I copy & pasted the wrong filepath 🙃 ddf-ui/ui-backend/catalog-ui-search/src/main/resources/OSGI-INF/metatype/org.codice.ddf.catalog.ui.xml

private int maxFileSizeInMemory = 50 * 1024 * 1024; // 50 MB

private List<String> readOnly =
ImmutableList.of(
Expand Down Expand Up @@ -402,11 +403,11 @@ public void setResultShow(List<String> resultShow) {
this.resultShow = resultShow;
}

public void setMaximumUploadSize(int size) {
public void setMaximumUploadSize(long size) {
this.maximumUploadSize = size;
}

public int getMaximumUploadSize() {
public long getMaximumUploadSize() {
return maximumUploadSize;
}

Expand Down Expand Up @@ -573,6 +574,8 @@ public Map<String, Object> getConfig() {
config.put("menuIconSrc", menuIconSrc);
config.put("customBranding", customBranding);
config.put("extra", extra);
config.put("maximumUploadSize", maximumUploadSize);
config.put("maxFileSizeInMemory", maxFileSizeInMemory);

return config;
}
Expand Down Expand Up @@ -1264,4 +1267,12 @@ public String getMenuIconSrc() {
public void setMenuIconSrc(String menuIconSrc) {
this.menuIconSrc = menuIconSrc;
}

public void setMaxFileSizeInMemory(int size) {
this.maxFileSizeInMemory = size;
}

public int getMaxFileSizeInMemory() {
return maxFileSizeInMemory;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package org.codice.ddf.catalog.ui.util.jaxrs;

import java.util.*;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't do star imports.

import javax.ws.rs.core.Cookie;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import spark.Request;

public class JaxRsHttpHeaders implements HttpHeaders {
private static final Logger LOGGER = LoggerFactory.getLogger(JaxRsHttpHeaders.class);
private final Request sparkRequest;

public JaxRsHttpHeaders(Request sparkRequest) {
this.sparkRequest = sparkRequest;
}

@Override
public List<String> getRequestHeader(String name) {
String header = sparkRequest.headers(name);
return header != null ? Collections.singletonList(header) : Collections.emptyList();
}

@Override
public String getHeaderString(String s) {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getHeaderString()");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it make sense to throw a "NotImplementedException"?

return "";
}

@Override
public MultivaluedMap<String, String> getRequestHeaders() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getRequestHeaders()");
return new MultivaluedHashMap<>();
}

@Override
public List<MediaType> getAcceptableMediaTypes() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getAcceptableMediaTypes()");
return List.of();
}

@Override
public List<Locale> getAcceptableLanguages() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getAcceptableLanguages()");
return List.of();
}

@Override
public MediaType getMediaType() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getMediaType()");
return null;
}

@Override
public Locale getLanguage() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getLanguage()");
return null;
}

@Override
public Map<String, Cookie> getCookies() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getCookies()");
return Map.of();
}

@Override
public Date getDate() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getDate()");
return null;
}

@Override
public int getLength() {
LOGGER.warn("Not implemented: JaxRsHttpHeaders.getLength()");
return 0;
}
}
Loading