Added a custom repository method for indexation with "meilisearch:import" command

src/Command/IndexCommand.php

@@ -36,6 +36,11 @@ protected function getIndices(): Collection
             });
     }
 
+    protected function getIndexNameWithoutPrefix(string $prefixedIndexName): string
+    {
+        return preg_replace(\sprintf('/^%s/', preg_quote($this->prefix)), '', $prefixedIndexName) ?? $prefixedIndexName;
+    }
+
     protected function getEntitiesFromArgs(InputInterface $input, OutputInterface $output): Collection
     {
         $indices = $this->getIndices();

src/Command/MeilisearchImportCommand.php

@@ -6,6 +6,7 @@
 
 use Doctrine\Persistence\ManagerRegistry;
 use Meilisearch\Bundle\Collection;
+use Meilisearch\Bundle\DataProvider\DoctrineOrmDataProvider;
 use Meilisearch\Bundle\EventListener\ConsoleOutputSubscriber;
 use Meilisearch\Bundle\Exception\TaskException;
 use Meilisearch\Bundle\Model\Aggregator;
@@ -99,10 +100,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $totalIndexed = 0;
 
             $manager = $this->managerRegistry->getManagerForClass($entityClassName);
-            $repository = $manager->getRepository($entityClassName);
-            $classMetadata = $manager->getClassMetadata($entityClassName);
-            $entityIdentifiers = $classMetadata->getIdentifierFieldNames();
-            $sortByAttrs = array_combine($entityIdentifiers, array_fill(0, \count($entityIdentifiers), 'ASC'));
 
             $output->writeln('<info>Importing for index '.$entityClassName.'</info>');
 
@@ -119,12 +116,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
 
             do {
-                $entities = $repository->findBy(
-                    [],
-                    $sortByAttrs,
-                    $batchSize,
-                    $batchSize * $page
-                );
+                $entities = $this->getEntities($index['name'], $entityClassName, $batchSize, $page);
 
                 $responses = $this->formatIndexingResponse($this->searchService->index($manager, $entities), $responseTimeout);
                 $totalIndexed += \count($entities);
@@ -208,4 +200,24 @@ private function entitiesToIndex($indexes): array
 
         return array_unique($indexes->all(), SORT_REGULAR);
     }
+
+    /**
+     * @param string $prefixedIndexName
+     * @param string $entityClassName
+     * @param int    $batchSize
+     * @param int    $page
+     *
+     * @return array
+     */
+    private function getEntities($prefixedIndexName, $entityClassName, $batchSize, $page): array
+    {
+        $dataProvider = $this->searchService->getDataProvider($this->getIndexNameWithoutPrefix($prefixedIndexName));
+
+        if (null === $dataProvider) {
+            $dataProvider = new DoctrineOrmDataProvider($this->managerRegistry);
+            $dataProvider->setEntityClassName($entityClassName);
+        }
+
+        return $dataProvider->getAll($batchSize, $batchSize * $page);
+    }
 }

src/DataProvider/DataProvider.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Meilisearch\Bundle\DataProvider;
+
+interface DataProvider
+{
+    /**
+     * Returns every objects that need to be indexed.
+     *
+     * @param int $limit
+     * @param int $offset
+     *
+     * @return array
+     */
+    public function getAll(int $limit = 100, int $offset = 0): array;
+}

src/DataProvider/DoctrineOrmDataProvider.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Meilisearch\Bundle\DataProvider;
+
+use Doctrine\Persistence\ManagerRegistry;
+
+final class DoctrineOrmDataProvider implements DataProvider
+{
+    private string $entityClassName;
+    private ManagerRegistry $managerRegistry;
+
+    public function __construct(ManagerRegistry $managerRegistry)
+    {
+        $this->managerRegistry = $managerRegistry;
+    }
+
+    public function setEntityClassName(string $entityClassName): void
+    {
+        $this->entityClassName = $entityClassName;
+    }
+
+    public function getAll(int $limit = 100, int $offset = 0): array
+    {
+        if (empty($this->entityClassName)) {
+            throw new \Exception('No entity class name set on data provider.');
+        }
+
+        $manager = $this->managerRegistry->getManagerForClass($this->entityClassName);
+        $classMetadata = $manager->getClassMetadata($this->entityClassName);
+        $entityIdentifiers = $classMetadata->getIdentifierFieldNames();
+        $repository = $manager->getRepository($this->entityClassName);
+        $sortByAttrs = array_combine($entityIdentifiers, array_fill(0, \count($entityIdentifiers), 'ASC'));
+
+        return $repository->findBy(
+            [],
+            $sortByAttrs,
+            $limit,
+            $offset
+        );
+    }
+}

src/DependencyInjection/Configuration.php

@@ -59,6 +59,10 @@ public function getConfigTreeBuilder(): TreeBuilder
                                 ->info('Property accessor path (like method or property name) used to decide if an entry should be indexed.')
                                 ->defaultNull()
                             ->end()
+                            ->scalarNode('data_provider')
+                                ->info('Method of the entity repository called when the meilisearch:import command is invoked.')
+                                ->defaultNull()
+                            ->end()
                             ->arrayNode('settings')
                                 ->info('Configure indices settings, see: https://www.meilisearch.com/docs/reference/api/settings')
                                 ->beforeNormalization()

src/DependencyInjection/MeilisearchExtension.php

@@ -30,6 +30,10 @@ public function load(array $configs, ContainerBuilder $container): void
         foreach ($config['indices'] as $index => $indice) {
             $config['indices'][$index]['prefixed_name'] = $config['prefix'].$indice['name'];
             $config['indices'][$index]['settings'] = $this->findReferences($config['indices'][$index]['settings']);
+
+            if (null !== $config['indices'][$index]['data_provider']) {
+                $config['indices'][$index]['data_provider'] = new Reference($config['indices'][$index]['data_provider']);
+            }
         }
 
         $container->setParameter('meili_url', $config['url'] ?? null);

src/SearchService.php

@@ -5,6 +5,7 @@
 namespace Meilisearch\Bundle;
 
 use Doctrine\Persistence\ObjectManager;
+use Meilisearch\Bundle\DataProvider\DataProvider;
 
 interface SearchService
 {
@@ -79,4 +80,6 @@ public function rawSearch(
      * @return int<0, max>
      */
     public function count(string $className, string $query = '', array $searchParams = []): int;
+
+    public function getDataProvider(string $indexName): ?DataProvider;
 }

src/Services/MeilisearchService.php

@@ -8,6 +8,7 @@
 use Doctrine\ORM\Proxy\DefaultProxyClassNameResolver;
 use Doctrine\Persistence\ObjectManager;
 use Meilisearch\Bundle\Collection;
+use Meilisearch\Bundle\DataProvider\DataProvider;
 use Meilisearch\Bundle\Engine;
 use Meilisearch\Bundle\Entity\Aggregator;
 use Meilisearch\Bundle\Exception\ObjectIdNotFoundException;
@@ -42,6 +43,7 @@ final class MeilisearchService implements SearchService
      */
     private array $classToSerializerGroup;
     private array $indexIfMapping;
+    private array $dataProviderMapping;
 
     public function __construct(NormalizerInterface $normalizer, Engine $engine, array $configuration, ?PropertyAccessorInterface $propertyAccessor = null)
     {
@@ -54,6 +56,7 @@ public function __construct(NormalizerInterface $normalizer, Engine $engine, arr
         $this->setAggregatorsAndEntitiesAggregators();
         $this->setClassToSerializerGroup();
         $this->setIndexIfMapping();
+        $this->setDataProviderMapping();
     }
 
     public function isSearchable($className): bool
@@ -223,6 +226,11 @@ public function shouldBeIndexed(object $entity): bool
         return true;
     }
 
+    public function getDataProvider(string $indexName): ?DataProvider
+    {
+        return $this->dataProviderMapping[$indexName] ?? null;
+    }
+
     /**
      * @param object|class-string $objectOrClass
      *
@@ -295,6 +303,17 @@ private function setIndexIfMapping(): void
         $this->indexIfMapping = $mapping;
     }
 
+    private function setDataProviderMapping(): void
+    {
+        $mapping = [];
+
+        /** @var array $indexDetails */
+        foreach ($this->configuration->get('indices') as $indexDetails) {
+            $mapping[$indexDetails['name']] = $indexDetails['data_provider'];
+        }
+        $this->dataProviderMapping = $mapping;
+    }
+
     /**
      * Returns the aggregators instances of the provided entities.
      *

tests/BaseKernelTestCase.php

@@ -18,6 +18,7 @@
 use Meilisearch\Bundle\Tests\Entity\Podcast;
 use Meilisearch\Bundle\Tests\Entity\Post;
 use Meilisearch\Bundle\Tests\Entity\Tag;
+use Meilisearch\Bundle\Tests\Entity\Ticket;
 use Meilisearch\Client;
 use Meilisearch\Exceptions\ApiException;
 use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase;
@@ -194,6 +195,16 @@ protected function createLink(array $properties = []): Link
         return $link;
     }
 
+    protected function createTicket(int $id, bool $sold): Ticket
+    {
+        $ticket = new Ticket($id, str_pad((string) random_int(0, 1000000), 6, '0', STR_PAD_LEFT), $sold);
+
+        $this->entityManager->persist($ticket);
+        $this->entityManager->flush();
+
+        return $ticket;
+    }
+
     protected function getPrefix(): string
     {
         return $this->searchService->getConfiguration()->get('prefix');

tests/DataProvider/TicketDataProvider.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Meilisearch\Bundle\Tests\DataProvider;
+
+use Doctrine\Persistence\ManagerRegistry;
+use Meilisearch\Bundle\DataProvider\DataProvider;
+use Meilisearch\Bundle\Tests\Entity\Ticket;
+
+final class TicketDataProvider implements DataProvider
+{
+    private ManagerRegistry $managerRegistry;
+
+    public function __construct(ManagerRegistry $managerRegistry)
+    {
+        $this->managerRegistry = $managerRegistry;
+    }
+
+    public function getAll(int $limit = 100, int $offset = 0): array
+    {
+        $manager = $this->managerRegistry->getManagerForClass(Ticket::class);
+        $repository = $manager->getRepository(Ticket::class);
+
+        return $repository->findBy(
+            ['sold' => false],
+            ['id' => 'ASC'],
+            $limit,
+            $offset
+        );
+    }
+}

tests/Entity/Ticket.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Meilisearch\Bundle\Tests\Entity;
+
+use Doctrine\DBAL\Types\Types;
+use Doctrine\ORM\Mapping as ORM;
+
+/**
+ * @ORM\Entity
+ */
+#[ORM\Entity]
+class Ticket
+{
+    /**
+     * @ORM\Id
+     *
+     * @ORM\Column(type="integer")
+     */
+    #[ORM\Id]
+    #[ORM\Column(type: Types::INTEGER)]
+    private int $id;
+
+    /**
+     * @ORM\Column(type="string")
+     */
+    #[ORM\Column(type: Types::STRING)]
+    private string $barcode;
+
+    /**
+     * @ORM\Column(type="boolean", options={"default"=false})
+     */
+    #[ORM\Column(type: Types::BOOLEAN, options: ['default' => false])]
+    private bool $sold;
+
+    /**
+     * @param int    $id
+     * @param string $barcode
+     * @param bool   $sold
+     */
+    public function __construct(int $id, string $barcode, bool $sold)
+    {
+        $this->id = $id;
+        $this->barcode = $barcode;
+        $this->sold = $sold;
+    }
+
+    public function getId(): int
+    {
+        return $this->id;
+    }
+
+    public function setId(int $id): Ticket
+    {
+        $this->id = $id;
+
+        return $this;
+    }
+
+    public function getBarcode(): string
+    {
+        return $this->barcode;
+    }
+
+    public function setBarcode(string $barcode): Ticket
+    {
+        $this->barcode = $barcode;
+
+        return $this;
+    }
+
+    public function isSold(): bool
+    {
+        return $this->sold;
+    }
+
+    public function setSold(bool $sold): Ticket
+    {
+        $this->sold = $sold;
+
+        return $this;
+    }
+}

tests/Integration/Command/MeilisearchClearCommandTest.php

@@ -36,6 +36,7 @@ public function testClear(): void
 Cleared sf_phpunit__self_normalizable index of Meilisearch\Bundle\Tests\Entity\SelfNormalizable
 Cleared sf_phpunit__dummy_custom_groups index of Meilisearch\Bundle\Tests\Entity\DummyCustomGroups
 Cleared sf_phpunit__dynamic_settings index of Meilisearch\Bundle\Tests\Entity\DynamicSettings
+Cleared sf_phpunit__tickets index of Meilisearch\Bundle\Tests\Entity\Ticket
 Done!
 
 EOD, $commandTester->getDisplay());

tests/Integration/Command/MeilisearchCreateCommandTest.php

@@ -60,6 +60,7 @@ public function testWithoutIndices(bool $updateSettings): void
 Setting "searchableAttributes" updated of "sf_phpunit__dynamic_settings".
 Setting "stopWords" updated of "sf_phpunit__dynamic_settings".
 Setting "synonyms" updated of "sf_phpunit__dynamic_settings".
+Creating index sf_phpunit__tickets for Meilisearch\Bundle\Tests\Entity\Ticket
 Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Post
 Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Tag
 Done!
@@ -76,6 +77,7 @@ public function testWithoutIndices(bool $updateSettings): void
 Creating index sf_phpunit__self_normalizable for Meilisearch\Bundle\Tests\Entity\SelfNormalizable
 Creating index sf_phpunit__dummy_custom_groups for Meilisearch\Bundle\Tests\Entity\DummyCustomGroups
 Creating index sf_phpunit__dynamic_settings for Meilisearch\Bundle\Tests\Entity\DynamicSettings
+Creating index sf_phpunit__tickets for Meilisearch\Bundle\Tests\Entity\Ticket
 Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Post
 Creating index sf_phpunit__aggregated for Meilisearch\Bundle\Tests\Entity\Tag
 Done!

tests/Integration/Command/MeilisearchDeleteCommandTest.php

@@ -37,6 +37,7 @@ public function testDeleteWithoutIndices(): void
 Deleted sf_phpunit__self_normalizable
 Deleted sf_phpunit__dummy_custom_groups
 Deleted sf_phpunit__dynamic_settings
+Deleted sf_phpunit__tickets
 Done!
 
 EOD, $clearOutput);

tests/Integration/Command/MeilisearchImportCommandTest.php

@@ -391,4 +391,34 @@ public function testAlias(): void
 
         self::assertSame(['meili:import'], $command->getAliases());
     }
+
+    public function testImportDataProvider(): void
+    {
+        $this->createTicket(1, true);
+        $this->createTicket(2, false);
+
+        $importCommand = $this->application->find('meilisearch:clear');
+        $importCommandTester = new CommandTester($importCommand);
+        $importCommandTester->execute(['--indices' => 'tickets']);
+
+        $importCommand = $this->application->find('meilisearch:import');
+        $importCommandTester = new CommandTester($importCommand);
+        $importCommandTester->execute(['--indices' => 'tickets']);
+
+        $importOutput = $importCommandTester->getDisplay();
+
+        $this->assertSame(<<<'EOD'
+Importing for index Meilisearch\Bundle\Tests\Entity\Ticket
+Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Ticket entities into sf_phpunit__tickets index (1 indexed since start)
+Done!
+
+EOD, $importOutput);
+
+        /** @var SearchResult $searchResult */
+        $searchResult = $this->client->index($this->getPrefix().'tickets')->search(null);
+
+        $this->assertEquals(1, $searchResult->getHitsCount());
+        $this->assertEquals(2, $searchResult->getHit(0)['id']);
+        $this->assertFalse($searchResult->getHit(0)['sold']);
+    }
 }

tests/Unit/ConfigurationTest.php

@@ -62,12 +62,18 @@ public function dataTestConfigurationTree(): array
                 [
                     'prefix' => 'sf_',
                     'indices' => [
-                        ['name' => 'posts', 'class' => 'App\Entity\Post', 'index_if' => null],
+                        [
+                            'name' => 'posts',
+                            'class' => 'App\Entity\Post',
+                            'index_if' => null,
+                            'data_provider' => null,
+                        ],
                         [
                             'name' => 'tags',
                             'class' => 'App\Entity\Tag',
                             'enable_serializer_groups' => true,
                             'index_if' => null,
+                            'data_provider' => null,
                         ],
                     ],
                 ],
@@ -86,6 +92,7 @@ public function dataTestConfigurationTree(): array
                             'serializer_groups' => ['searchable'],
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                         1 => [
                             'name' => 'tags',
@@ -94,6 +101,7 @@ public function dataTestConfigurationTree(): array
                             'serializer_groups' => ['searchable'],
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                     ],
                 ],
@@ -108,13 +116,15 @@ public function dataTestConfigurationTree(): array
                             'enable_serializer_groups' => false,
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                         [
                             'name' => 'items',
                             'class' => 'App\Entity\Tag',
                             'enable_serializer_groups' => false,
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                     ],
                     'nbResults' => 20,
@@ -131,7 +141,9 @@ public function dataTestConfigurationTree(): array
                             'class' => 'App\Entity\Post',
                             'enable_serializer_groups' => false,
                             'serializer_groups' => ['searchable'],
-                            'index_if' => null, 'settings' => [],
+                            'index_if' => null,
+                            'settings' => [],
+                            'data_provider' => null,
                         ],
                         [
                             'name' => 'items',
@@ -140,6 +152,7 @@ public function dataTestConfigurationTree(): array
                             'serializer_groups' => ['searchable'],
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                     ],
                     'nbResults' => 20,
@@ -159,6 +172,7 @@ public function dataTestConfigurationTree(): array
                             'serializer_groups' => ['post.public', 'post.private'],
                             'index_if' => null,
                             'settings' => [],
+                            'data_provider' => null,
                         ],
                     ],
                 ],
@@ -171,7 +185,9 @@ public function dataTestConfigurationTree(): array
                             'class' => 'App\Entity\Post',
                             'enable_serializer_groups' => true,
                             'serializer_groups' => ['post.public', 'post.private'],
-                            'index_if' => null, 'settings' => [],
+                            'index_if' => null,
+                            'settings' => [],
+                            'data_provider' => null,
                         ],
                     ],
                     'nbResults' => 20,
@@ -206,6 +222,7 @@ public function dataTestConfigurationTree(): array
                             'settings' => [
                                 'distinctAttribute' => ['product_id'],
                             ],
+                            'data_provider' => null,
                         ],
                     ],
                     'nbResults' => 20,
@@ -240,6 +257,38 @@ public function dataTestConfigurationTree(): array
                             'settings' => [
                                 'proximityPrecision' => ['byWord'],
                             ],
+                            'data_provider' => null,
+                        ],
+                    ],
+                    'nbResults' => 20,
+                    'batchSize' => 500,
+                    'serializer' => 'serializer',
+                    'doctrineSubscribedEvents' => ['postPersist', 'postUpdate', 'preRemove'],
+                ],
+            ],
+            'custom data provider' => [
+                [
+                    'prefix' => 'sf_',
+                    'indices' => [
+                        [
+                            'name' => 'items',
+                            'class' => 'App\Entity\Post',
+                            'data_provider' => 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider',
+                        ],
+                    ],
+                ],
+                [
+                    'url' => 'http://localhost:7700',
+                    'prefix' => 'sf_',
+                    'indices' => [
+                        [
+                            'name' => 'items',
+                            'class' => 'App\Entity\Post',
+                            'enable_serializer_groups' => false,
+                            'serializer_groups' => ['searchable'],
+                            'index_if' => null,
+                            'settings' => [],
+                            'data_provider' => 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider',
                         ],
                     ],
                     'nbResults' => 20,

tests/Unit/DataProvider/DoctrineOrmDataProviderTest.php

@@ -0,0 +1,24 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Meilisearch\Bundle\Tests\Unit\DataProvider;
+
+use Meilisearch\Bundle\DataProvider\DoctrineOrmDataProvider;
+use Meilisearch\Bundle\Tests\BaseKernelTestCase;
+
+/**
+ * Class ConfigurationTest.
+ */
+class DoctrineOrmDataProviderTest extends BaseKernelTestCase
+{
+    public function testDefaultDataProviderWithoutEntityClassName(): void
+    {
+        $dataProvider = new DoctrineOrmDataProvider($this->get('doctrine'));
+
+        $this->expectException(\Exception::class);
+        $this->expectExceptionMessage('No entity class name set on data provider.');
+
+        $dataProvider->getAll();
+    }
+}

tests/config/config.yaml

@@ -24,3 +24,7 @@ doctrine:
                 dir: '%kernel.project_dir%/tests/Entity'
                 prefix: 'Meilisearch\Bundle\Tests\Entity'
                 alias: App
+
+services:
+    Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider:
+        arguments: [ '@doctrine' ]

tests/config/config_php7.yaml

@@ -28,3 +28,7 @@ doctrine:
                 dir: '%kernel.project_dir%/tests/Entity'
                 prefix: 'Meilisearch\Bundle\Tests\Entity'
                 alias: App
+
+services:
+    Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider:
+        arguments: [ '@doctrine' ]

tests/config/meilisearch.yaml

@@ -54,6 +54,9 @@ meilisearch:
           _service: '@Meilisearch\Bundle\Tests\Integration\Fixtures\StopWords'
         synonyms:
           _service: '@Meilisearch\Bundle\Tests\Integration\Fixtures\Synonyms'
+    - name: tickets
+      class: 'Meilisearch\Bundle\Tests\Entity\Ticket'
+      data_provider: 'Meilisearch\Bundle\Tests\DataProvider\TicketDataProvider'
 
 services:
   Meilisearch\Bundle\Tests\Integration\Fixtures\: