diff --git a/core/src/main/java/jeeves/transaction/BatchItemProcessor.java b/core/src/main/java/jeeves/transaction/BatchItemProcessor.java new file mode 100644 index 000000000000..494ce330d7dd --- /dev/null +++ b/core/src/main/java/jeeves/transaction/BatchItemProcessor.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2001-2026 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package jeeves.transaction; + +/** + * Interface for processing an item in a batch. + * + * @param The type of the item to process. + */ +@FunctionalInterface +public interface BatchItemProcessor { + /** + * Process an item. + * + * @param item The item to process. + * @throws Exception If an error occurs during processing. + */ + void process(T item) throws Exception; +} diff --git a/core/src/main/java/jeeves/transaction/BatchTransactionalProcessor.java b/core/src/main/java/jeeves/transaction/BatchTransactionalProcessor.java new file mode 100644 index 000000000000..c0189642374f --- /dev/null +++ b/core/src/main/java/jeeves/transaction/BatchTransactionalProcessor.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2001-2026 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package jeeves.transaction; + +import com.google.common.collect.Iterables; +import org.springframework.context.ApplicationContext; + +import java.util.List; + +/** + * Generic processor for processing items in batches within transactions. + *

+ * Each batch runs in its own {@code PROPAGATION_REQUIRES_NEW} transaction. Because of this, + * integration tests exercising code that uses this class cannot rely on the test framework's + * usual transaction-per-test rollback (e.g. Spring's {@code @Transactional} test rollback): + * the nested per-batch transactions commit independently of the outer test transaction. Such + * tests should run with {@code @Transactional(propagation = Propagation.NOT_SUPPORTED)} and + * clean up any created/modified data explicitly (e.g. in an {@code @After} method). + * + * @param The type of the items to process. + */ +public class BatchTransactionalProcessor { + + private final String transactionName; + private final ApplicationContext applicationContext; + private int batchSize = 100; + + /** + * Constructor. + * + * @param transactionName The name of the transaction. + * @param applicationContext The application context. + */ + public BatchTransactionalProcessor(String transactionName, ApplicationContext applicationContext) { + this.transactionName = transactionName; + this.applicationContext = applicationContext; + } + + /** + * Sets the batch size. + * + * @param batchSize The batch size. Must be greater than 0. + * @return This processor. + */ + public BatchTransactionalProcessor setBatchSize(int batchSize) { + if (batchSize <= 0) { + throw new IllegalArgumentException("batchSize must be greater than 0"); + } + this.batchSize = batchSize; + return this; + } + + /** + * Process items in batches. + * + * @param items The items to process. + * @param itemProcessor The processor for each item. + * @throws Exception If an error occurs during processing. + */ + public void process(Iterable items, BatchItemProcessor itemProcessor) throws Exception { + Iterable> batches = Iterables.partition(items, batchSize); + + for (final List batch : batches) { + TransactionManager.runInTransaction(transactionName, applicationContext, + TransactionManager.TransactionRequirement.CREATE_NEW, + TransactionManager.CommitBehavior.ALWAYS_COMMIT, false, transactionStatus -> { + for (T item : batch) { + itemProcessor.process(item); + } + return null; + }); + } + } +} diff --git a/services/src/main/java/org/fao/geonet/api/processing/XslProcessApi.java b/services/src/main/java/org/fao/geonet/api/processing/XslProcessApi.java index f45d8d389704..164eb5705c95 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/XslProcessApi.java +++ b/services/src/main/java/org/fao/geonet/api/processing/XslProcessApi.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * Copyright (C) 2001-2026 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -61,6 +61,8 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; +import jeeves.transaction.BatchTransactionalProcessor; + import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; @@ -102,12 +104,10 @@ public class XslProcessApi { @Autowired SchemaManager schemaMan; - - @Autowired - private IMetadataManager metadataManager; - @Autowired SettingManager settingManager; + @Autowired + private IMetadataManager metadataManager; @io.swagger.v3.oas.annotations.Operation( summary = "Preview process result applied to one or more records", @@ -135,50 +135,44 @@ public Object previewProcessRecords( description = ApiParams.API_PARAM_PROCESS_ID ) @PathVariable - String process, + String process, @Parameter( - description = "Return differences with diff, diffhtml or patch", - required = false + description = "Return differences with diff, diffhtml or patch" ) @RequestParam( required = false ) - DiffType diffType, - @Parameter(description = API_PARAM_RECORD_UUIDS_OR_SELECTION, - required = false, - example = "") + DiffType diffType, + @Parameter(description = API_PARAM_RECORD_UUIDS_OR_SELECTION) @RequestParam(required = false) - String[] uuids, + String[] uuids, @Parameter( - description = ApiParams.API_PARAM_BUCKET_NAME, - required = false) + description = ApiParams.API_PARAM_BUCKET_NAME) @RequestParam( required = false ) - String bucket, + String bucket, @Parameter(description = "Append documents before processing", - required = false, example = "false") @RequestParam(required = false, defaultValue = "false") - boolean appendFirst, + boolean appendFirst, @Parameter(description = "Apply update fixed info", - required = false, example = "false") @RequestParam(required = false, defaultValue = "true") boolean applyUpdateFixedInfo, @Parameter(hidden = true) - HttpSession httpSession, + HttpSession httpSession, @Parameter(hidden = true) - HttpServletRequest request, + HttpServletRequest request, @Parameter(hidden = true) - HttpServletResponse response) throws IllegalArgumentException { + HttpServletResponse response) throws IllegalArgumentException { UserSession session = ApiUtils.getUserSession(httpSession); XsltMetadataProcessingReport xslProcessingReport = new XsltMetadataProcessingReport(process); Element preview = new Element("preview"); - StringBuffer output = new StringBuffer(); + StringBuilder output = new StringBuilder(); boolean isText = process.endsWith(".csv"); @@ -224,7 +218,7 @@ public Object previewProcessRecords( if (record != null) { if (applyUpdateFixedInfo) { record = metadataManager.updateFixedInfo(dataMan.getMetadataSchema(id), - Optional.absent(), uuid, record, null, UpdateDatestamp.NO, serviceContext); + Optional.absent(), uuid, record, null, UpdateDatestamp.NO, serviceContext); } if (diffType != null) { IMetadataUtils metadataUtils = serviceContext.getBean(IMetadataUtils.class); @@ -256,7 +250,7 @@ record = metadataManager.updateFixedInfo(dataMan.getMetadataSchema(id), } // In case of errors during processing return report. - if (xslProcessingReport.getErrors().size() > 0) { + if (!xslProcessingReport.getErrors().isEmpty()) { response.setHeader(CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); ObjectMapper mapper = new ObjectMapper(); try { @@ -291,37 +285,33 @@ public XsltMetadataProcessingReport processRecords( description = ApiParams.API_PARAM_PROCESS_ID ) @PathVariable - String process, - @Parameter(description = API_PARAM_RECORD_UUIDS_OR_SELECTION, - required = false, - example = "") + String process, + @Parameter(description = API_PARAM_RECORD_UUIDS_OR_SELECTION) @RequestParam(required = false) - String[] uuids, + String[] uuids, @Parameter( - description = ApiParams.API_PARAM_BUCKET_NAME, - required = false) + description = ApiParams.API_PARAM_BUCKET_NAME + ) @RequestParam( required = false ) - String bucket, + String bucket, @Parameter( - description = ApiParams.API_PARAM_UPDATE_DATESTAMP, - required = false + description = ApiParams.API_PARAM_UPDATE_DATESTAMP ) @RequestParam( required = false, defaultValue = "true" ) - boolean updateDateStamp, + boolean updateDateStamp, @Parameter(description = "Index after processing", - required = false, example = "false") @RequestParam(required = false, defaultValue = "true") - boolean index, + boolean index, @Parameter(hidden = true) - HttpSession httpSession, + HttpSession httpSession, @Parameter(hidden = true) - HttpServletRequest request) throws Exception { + HttpServletRequest request) throws Exception { UserSession session = ApiUtils.getUserSession(httpSession); XsltMetadataProcessingReport xslProcessingReport = @@ -387,11 +377,13 @@ public BatchXslMetadataReindexer(ServiceContext context, @Override public void process(String catalogueId) throws Exception { - DataManager dataMan = context.getBean(DataManager.class); IMetadataUtils metadataUtils = context.getBean(IMetadataUtils.class); ApplicationContext appContext = ApplicationContextHolder.get(); - for (String uuid : this.records) { + + BatchTransactionalProcessor processor = + new BatchTransactionalProcessor<>("BatchXslProcess", appContext); + processor.process(this.records, uuid -> { List idList = metadataUtils.findAllIdsBy(MetadataSpecs.hasMetadataUuid(uuid)); // Increase the total records counter when processing a metadata with approved and working copies @@ -401,23 +393,35 @@ public void process(String catalogueId) throws Exception { } for (Integer id : idList) { - Log.info("org.fao.geonet.services.metadata", - "Processing metadata with id:" + id); - - Element beforeMetadata = dataMan.getMetadata(context, String.valueOf(id), false, false, false); - - XslProcessUtils.process(context, String.valueOf(id), process, - true, index, updateDateStamp, xslProcessingReport, - siteURL, request.getParameterMap()); - - Element afterMetadata = dataMan.getMetadata(context, String.valueOf(id), false, false, false); - - XMLOutputter outp = new XMLOutputter(); - String xmlAfter = outp.outputString(afterMetadata); - String xmlBefore = outp.outputString(beforeMetadata); - new RecordProcessingChangeEvent(id, this.userId, xmlBefore, xmlAfter, process).publish(appContext); + try { + Log.info("org.fao.geonet.services.metadata", + "Processing metadata with id:" + id); + + Element beforeMetadata = dm.getMetadata(context, String.valueOf(id), false, false, false); + + XslProcessUtils.process(context, String.valueOf(id), process, + true, index, updateDateStamp, xslProcessingReport, + siteURL, request.getParameterMap()); + + Element afterMetadata = dm.getMetadata(context, String.valueOf(id), false, false, false); + + XMLOutputter outp = new XMLOutputter(); + String xmlAfter = outp.outputString(afterMetadata); + String xmlBefore = outp.outputString(beforeMetadata); + new RecordProcessingChangeEvent(id, userId, xmlBefore, xmlAfter, process).publish(appContext); + } catch (Exception e) { + // Isolate the failure to this record so the rest of the batch (and + // subsequent batches) still get processed instead of being rolled back + // or skipped, see BatchTransactionalProcessor. + AbstractMetadata metadata = metadataUtils.findOne(id); + if (metadata != null) { + xslProcessingReport.addMetadataError(metadata, e); + } else { + xslProcessingReport.addMetadataError(id, uuid, false, false, e); + } + } } - } + }); } } } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/BatchEditsApi.java b/services/src/main/java/org/fao/geonet/api/records/editing/BatchEditsApi.java index 3ea7d27819f7..dde3477cfd9d 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/BatchEditsApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/BatchEditsApi.java @@ -33,6 +33,7 @@ import jeeves.server.UserSession; import jeeves.server.context.ServiceContext; import jeeves.services.ReadWriteController; +import jeeves.transaction.BatchTransactionalProcessor; import org.apache.commons.lang.StringUtils; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.api.ApiParams; @@ -213,11 +214,13 @@ private Pair applyBatchEdits( report.setTotalRecords(setOfUuidsToEdit.size()); UserSession userSession = ApiUtils.getUserSession(request.getSession()); - String changeDate = null; Element preview = new Element("preview"); final IMetadataUtils metadataRepository = context.getBean(IMetadataUtils.class); - for (String recordUuid : setOfUuidsToEdit) { + + BatchTransactionalProcessor processor = + new BatchTransactionalProcessor<>("BatchEditing", appContext); + processor.process(setOfUuidsToEdit, recordUuid -> { AbstractMetadata record = metadataRepository.findOneByUuid(recordUuid); if (record == null) { report.incrementNullRecords(); @@ -264,7 +267,7 @@ private Pair applyBatchEdits( } if (previewOnly) { if (diffType == null) { - preview.addContent(metadata); + preview.addContent(metadata.detach()); } else { preview.addContent( Diff.diff(original, Xml.getString(metadata), diffType) @@ -280,7 +283,7 @@ private Pair applyBatchEdits( serviceContext, record.getId() + "", metadata, validate, ufo, "eng", // Not used when validate is false - changeDate, uds, IndexingMode.full); + null, uds, IndexingMode.full); report.addMetadataInfos(record, "Metadata updated."); Element afterMetadata = dataMan.getMetadata(serviceContext, String.valueOf(record.getId()), false, false, false); @@ -296,7 +299,7 @@ private Pair applyBatchEdits( } report.incrementProcessedRecords(); } - } + }); report.close(); return Pair.write(report, preview); } diff --git a/services/src/test/java/org/fao/geonet/api/processing/XslProcessApiTest.java b/services/src/test/java/org/fao/geonet/api/processing/XslProcessApiTest.java new file mode 100644 index 000000000000..1c3a9135a60c --- /dev/null +++ b/services/src/test/java/org/fao/geonet/api/processing/XslProcessApiTest.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2001-2026 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ +package org.fao.geonet.api.processing; + +import com.google.common.collect.Sets; +import jeeves.server.context.ServiceContext; +import org.fao.geonet.ApplicationContextHolder; +import org.fao.geonet.api.processing.report.XsltMetadataProcessingReport; +import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpSession; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.*; + +public class XslProcessApiTest { + + private ServiceContext serviceContext; + private DataManager dataManager; + private IMetadataUtils metadataUtils; + private ConfigurableApplicationContext applicationContext; + private PlatformTransactionManager transactionManager; + + @Before + public void setUp() { + serviceContext = mock(ServiceContext.class); + dataManager = mock(DataManager.class); + metadataUtils = mock(IMetadataUtils.class); + applicationContext = mock(ConfigurableApplicationContext.class); + transactionManager = mock(PlatformTransactionManager.class); + + when(serviceContext.getBean(DataManager.class)).thenReturn(dataManager); + when(serviceContext.getBean(IMetadataUtils.class)).thenReturn(metadataUtils); + when(applicationContext.getBean(PlatformTransactionManager.class)).thenReturn(transactionManager); + + // Mock transaction status + TransactionStatus transactionStatus = mock(TransactionStatus.class); + when(transactionManager.getTransaction(any())).thenReturn(transactionStatus); + + ApplicationContextHolder.set(applicationContext); + } + + @After + public void tearDown() { + ApplicationContextHolder.clear(); + } + + @Test + public void testBatchProcessing() throws Exception { + Set records = Sets.newHashSet(); + for (int i = 0; i < 150; i++) { + records.add("uuid-" + i); + } + + String process = "test-process"; + HttpSession session = mock(HttpSession.class); + String siteURL = "http://localhost:8080/geonetwork"; + XsltMetadataProcessingReport report = new XsltMetadataProcessingReport(process); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameterMap()).thenReturn(Collections.emptyMap()); + + // Mock metadataUtils to return an empty list for each UUID + when(metadataUtils.findAllIdsBy(any())).thenReturn(Collections.emptyList()); + + XslProcessApi.BatchXslMetadataReindexer reindexer = new XslProcessApi.BatchXslMetadataReindexer( + serviceContext, dataManager, records, process, session, siteURL, report, request, true, true, 1 + ); + + reindexer.process("catalogue"); + + // Verify that TransactionManager.runInTransaction was effectively called twice + // We check this by verifying that transactionManager.getTransaction was called twice + // since runInTransaction calls it once per batch. + verify(transactionManager, times(2)).getTransaction(any()); + + // Verify that findAllIdsBy was called for each record (150 times) + verify(metadataUtils, times(150)).findAllIdsBy(any()); + } + + @Test + public void testBatchProcessingContinuesAfterRecordFailure() throws Exception { + Set records = Sets.newHashSet(); + for (int i = 0; i < 150; i++) { + records.add("uuid-" + i); + } + + String process = "test-process"; + HttpSession session = mock(HttpSession.class); + String siteURL = "http://localhost:8080/geonetwork"; + XsltMetadataProcessingReport report = new XsltMetadataProcessingReport(process); + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getParameterMap()).thenReturn(Collections.emptyMap()); + + // Assign each uuid a distinct metadata id, in call order + AtomicInteger idSequence = new AtomicInteger(0); + when(metadataUtils.findAllIdsBy(any())).thenAnswer(invocation -> Collections.singletonList(idSequence.getAndIncrement())); + + // Every record fails as soon as its metadata is fetched, before any real XSL + // processing is attempted, so this stays a fast, dependency-free unit test. + when(dataManager.getMetadata(any(), anyString(), anyBoolean(), anyBoolean(), anyBoolean())) + .thenThrow(new RuntimeException("Simulated failure fetching metadata")); + when(metadataUtils.findOne(anyInt())).thenReturn(null); + + XslProcessApi.BatchXslMetadataReindexer reindexer = new XslProcessApi.BatchXslMetadataReindexer( + serviceContext, dataManager, records, process, session, siteURL, report, request, true, true, 1 + ); + + // Must not throw: a failure on one record must not abort the whole run. + reindexer.process("catalogue"); + + // Every uuid must still be looked up, including those in batches after the first failure. + verify(metadataUtils, times(150)).findAllIdsBy(any()); + + // Both batches (150 records / batch size 100) must still run their own transaction. + verify(transactionManager, times(2)).getTransaction(any()); + + // Every failure is captured in the report rather than silently dropped. + assertEquals(150, report.getNumberOfRecordsWithErrors()); + } +} diff --git a/services/src/test/java/org/fao/geonet/api/records/editing/BatchEditsApiTest.java b/services/src/test/java/org/fao/geonet/api/records/editing/BatchEditsApiTest.java new file mode 100644 index 000000000000..86e0d1cc5199 --- /dev/null +++ b/services/src/test/java/org/fao/geonet/api/records/editing/BatchEditsApiTest.java @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2001-2026 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ +package org.fao.geonet.api.records.editing; + +import jeeves.server.UserSession; +import jeeves.server.context.ServiceContext; +import org.fao.geonet.ApplicationContextHolder; +import org.fao.geonet.domain.AbstractMetadata; +import org.fao.geonet.domain.MetadataDataInfo; +import org.fao.geonet.kernel.AccessManager; +import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.SchemaManager; +import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.setting.SettingManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.security.access.hierarchicalroles.RoleHierarchy; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpSession; +import org.fao.geonet.api.tools.i18n.LanguageUtils; +import jeeves.server.dispatchers.ServiceManager; +import jeeves.constants.Jeeves; +import org.fao.geonet.kernel.BatchEditParameter; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import org.fao.geonet.api.processing.report.IProcessingReport; +import org.fao.geonet.api.processing.report.SimpleMetadataProcessingReport; +import org.fao.geonet.kernel.schema.MetadataSchema; +import org.fao.geonet.domain.Profile; +import java.util.*; + +public class BatchEditsApiTest { + + private BatchEditsApi api; + private ServiceContext serviceContext; + private DataManager dataManager; + private IMetadataUtils metadataUtils; + private ConfigurableApplicationContext applicationContext; + private PlatformTransactionManager transactionManager; + private SchemaManager schemaManager; + private AccessManager accessManager; + private SettingManager settingManager; + private RoleHierarchy roleHierarchy; + + private UserSession userSession; + private HttpSession session; + + private HttpServletRequest request; + + @Before + public void setUp() throws Exception { + api = new BatchEditsApi(); + serviceContext = mock(ServiceContext.class); + dataManager = mock(DataManager.class); + metadataUtils = mock(IMetadataUtils.class); + applicationContext = mock(ConfigurableApplicationContext.class); + transactionManager = mock(PlatformTransactionManager.class); + schemaManager = mock(SchemaManager.class); + accessManager = mock(AccessManager.class); + settingManager = mock(SettingManager.class); + roleHierarchy = mock(RoleHierarchy.class); + userSession = mock(UserSession.class); + session = mock(HttpSession.class); + request = mock(HttpServletRequest.class); + + api.setApplicationContext(applicationContext); + // We need to inject mocks into BatchEditsApi fields + java.lang.reflect.Field schemaManagerField = BatchEditsApi.class.getDeclaredField("_schemaManager"); + schemaManagerField.setAccessible(true); + schemaManagerField.set(api, schemaManager); + + java.lang.reflect.Field settingManagerField = BatchEditsApi.class.getDeclaredField("settingManager"); + settingManagerField.setAccessible(true); + settingManagerField.set(api, settingManager); + + java.lang.reflect.Field roleHierarchyField = BatchEditsApi.class.getDeclaredField("roleHierarchy"); + roleHierarchyField.setAccessible(true); + roleHierarchyField.set(api, roleHierarchy); + + when(applicationContext.getBean(DataManager.class)).thenReturn(dataManager); + when(applicationContext.getBean(SchemaManager.class)).thenReturn(schemaManager); + when(applicationContext.getBean(AccessManager.class)).thenReturn(accessManager); + when(applicationContext.getBean(IMetadataUtils.class)).thenReturn(metadataUtils); + when(applicationContext.getBean(PlatformTransactionManager.class)).thenReturn(transactionManager); + + // Mock transaction status + TransactionStatus transactionStatus = mock(TransactionStatus.class); + when(transactionManager.getTransaction(any())).thenReturn(transactionStatus); + + ApplicationContextHolder.set(applicationContext); + + LanguageUtils languageUtils = mock(LanguageUtils.class); + ServiceManager serviceManager = mock(ServiceManager.class); + when(applicationContext.getBean(LanguageUtils.class)).thenReturn(languageUtils); + when(applicationContext.getBean(ServiceManager.class)).thenReturn(serviceManager); + when(languageUtils.getIso3langCode(any())).thenReturn("eng"); + when(serviceManager.createServiceContext(anyString(), anyString(), any(HttpServletRequest.class))).thenReturn(serviceContext); + + when(serviceContext.getUserSession()).thenReturn(userSession); + when(userSession.getProfile()).thenReturn(Profile.Administrator); + + when(session.getAttribute(Jeeves.Elem.SESSION)).thenReturn(userSession); + when(request.getSession()).thenReturn(session); + + when(request.getLocales()).thenReturn(Collections.enumeration(Collections.singletonList(Locale.ENGLISH))); + } + + @After + public void tearDown() { + ApplicationContextHolder.clear(); + } + + @Test + public void testBatchProcessing() throws Exception { + MetadataSchema metadataSchema = mock(MetadataSchema.class); + when(schemaManager.getSchema(anyString())).thenReturn(metadataSchema); + when(metadataSchema.getNamespaces()).thenReturn(new java.util.ArrayList<>()); + + int numRecords = 150; + String[] uuids = new String[numRecords]; + for (int i = 0; i < numRecords; i++) { + uuids[i] = "uuid-" + i; + AbstractMetadata record = mock(AbstractMetadata.class); + when(record.getId()).thenReturn(i); + MetadataDataInfo dataInfo = mock(MetadataDataInfo.class); + when(record.getDataInfo()).thenReturn(dataInfo); + when(dataInfo.getSchemaId()).thenReturn("iso19139"); + when(record.getXmlData(false)).thenReturn(new org.jdom.Element("root")); + + when(metadataUtils.findOneByUuid(uuids[i])).thenReturn(record); + when(accessManager.isOwner(any(), anyString())).thenReturn(true); + } + + BatchEditParameter[] edits = new BatchEditParameter[1]; + edits[0] = new BatchEditParameter(); + edits[0].setXpath("/root/element"); + edits[0].setValue("value"); + + api.batchEdit(uuids, null, false, edits, request); + + // Verify that TransactionManager.runInTransaction was effectively called twice + verify(transactionManager, times(2)).getTransaction(any()); + } + + @Test + public void testBatchProcessingContinuesAfterRecordFailure() throws Exception { + when(schemaManager.getSchema(anyString())).thenThrow(new RuntimeException("Simulated schema failure")); + + int numRecords = 150; + String[] uuids = new String[numRecords]; + for (int i = 0; i < numRecords; i++) { + uuids[i] = "uuid-" + i; + AbstractMetadata record = mock(AbstractMetadata.class); + when(record.getId()).thenReturn(i); + MetadataDataInfo dataInfo = mock(MetadataDataInfo.class); + when(record.getDataInfo()).thenReturn(dataInfo); + when(dataInfo.getSchemaId()).thenReturn("iso19139"); + when(record.getXmlData(false)).thenReturn(new org.jdom.Element("root")); + + when(metadataUtils.findOneByUuid(uuids[i])).thenReturn(record); + when(accessManager.isOwner(any(), anyString())).thenReturn(true); + } + + BatchEditParameter[] edits = new BatchEditParameter[1]; + edits[0] = new BatchEditParameter(); + edits[0].setXpath("/root/element"); + edits[0].setValue("value"); + + // Must not throw: a failure processing one record must not abort the whole batch. + IProcessingReport report = api.batchEdit(uuids, null, false, edits, request); + + // Both batches (150 records / batch size 100) must still run their own transaction. + verify(transactionManager, times(2)).getTransaction(any()); + + // Every record failed, but every failure is captured rather than aborting the run. + SimpleMetadataProcessingReport simpleReport = (SimpleMetadataProcessingReport) report; + assertEquals(numRecords, simpleReport.getNumberOfRecordsWithErrors()); + assertEquals(numRecords, simpleReport.getNumberOfRecordsProcessed()); + } +} diff --git a/services/src/test/java/org/fao/geonet/services/metadata/BatchEditsServiceTest.java b/services/src/test/java/org/fao/geonet/services/metadata/BatchEditsServiceTest.java index 67ca12d6efb0..e106f1d1143c 100644 --- a/services/src/test/java/org/fao/geonet/services/metadata/BatchEditsServiceTest.java +++ b/services/src/test/java/org/fao/geonet/services/metadata/BatchEditsServiceTest.java @@ -37,12 +37,15 @@ import org.fao.geonet.kernel.BatchEditParameter; import org.fao.geonet.csw.common.util.Xml; import org.fao.geonet.domain.AbstractMetadata; +import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; import org.fao.geonet.kernel.mef.MEFLibIntegrationTest; +import org.fao.geonet.repository.specification.MetadataSpecs; import org.fao.geonet.schema.iso19115_3_2018.ISO19115_3_2018Namespaces; import org.fao.geonet.services.AbstractServiceIntegrationTest; import org.jdom.Attribute; import org.jdom.Element; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -59,6 +62,9 @@ import jeeves.server.context.ServiceContext; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + public class BatchEditsServiceTest extends AbstractServiceIntegrationTest { List uuids = new ArrayList(); @@ -71,12 +77,22 @@ public class BatchEditsServiceTest extends AbstractServiceIntegrationTest { @Autowired private IMetadataUtils repository; + @Autowired + private IMetadataManager metadataManager; + private MockMvc mockMvc; private MockHttpSession mockHttpSession; + @After + public void cleanup() { + metadataManager.deleteAll(MetadataSpecs.hasMetadataUuidIn(uuids)); + } @Before public void loadSamples() throws Exception { + uuids.clear(); + firstMetadataId = null; + context = createServiceContext(); loginAsAdmin(context); @@ -123,6 +139,7 @@ public void testParameterMustBeSet() throws Exception { @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecord() throws Exception { final BatchEditParameter[] listOfupdates = new BatchEditParameter[]{ new BatchEditParameter( @@ -165,6 +182,7 @@ public void testUpdateRecord() throws Exception { @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecordElementInsertWithParentXpath() throws Exception { final String uuid = "db07463b-6769-401e-944b-f22e2e3bcc26"; // XPath has no match and same type as fragment, element is created @@ -198,6 +216,7 @@ public void testUpdateRecordElementInsertWithParentXpath() throws Exception { @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecordUpdateAttribute() throws Exception { final String uuid = "db07463b-6769-401e-944b-f22e2e3bcc26"; BatchEditParameter[] listOfupdates = new BatchEditParameter[]{ @@ -291,6 +310,7 @@ public void testUpdateRecordDeleteAttribute() throws Exception { @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecordAddAttribute() throws Exception { final String uuid = "db07463b-6769-401e-944b-f22e2e3bcc26"; BatchEditParameter[] listOfupdates = new BatchEditParameter[]{ @@ -324,6 +344,7 @@ public void testUpdateRecordAddAttribute() throws Exception { } @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecordAddAndDeleteAttributeWithNamespace() throws Exception { final String uuid = "db07463b-6769-401e-944b-f22e2e3bcc26"; BatchEditParameter[] listOfupdates = new BatchEditParameter[]{ @@ -384,6 +405,7 @@ public void testUpdateRecordAddAndDeleteAttributeWithNamespace() throws Exceptio @Test + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void testUpdateRecordElement() throws Exception { final String uuid = "db07463b-6769-401e-944b-f22e2e3bcc26"; // XPath has no match and same type as fragment, element is created