Performance issues with Pulsar.

[ub-3]

We are currently in the process of migrating from gearmand to Apache Pulsar, but have run into performance issues. At the moment, we have several metrics on the application side, by which we determine the speed of message processing. The main metric with which we have problems is the speed of receiving a message from MQ, for example, 98 percentile for gearman is kept at 100-200 ms, while Pulsar is up to 1.2-1.3 sec. We tried to make a couple of optimizations in the configuration with Pulsar, but did not achieve any obvious success. If you have any suggestions for optimization, I will be very glad. Below I specified the Broker, Bookkeeper settings and the cluster configuration.

Environment:

• Ubuntu 20.04.5 LTS
• Mem: 27GB
• CPU: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz
• SSD 50GB
• Pulsar 2.11.0
• openjdk 17.0.5 2022-10-18

Usage:

• Topics count: ~4000
• 3-4 cosumers per topic
• Consumer: receive_queue_size: 1, Subscription: Shared

Cluster made of:

• 3 zk nodes
• 1 pulsar broker and 1 bookie server on a same machine.

BOOKIE_MEM="-Xms8g -Xmx8g -XX:MaxDirectMemorySize=8g"
PULSAR_MEM="-Xms8g -Xmx8g -XX:MaxDirectMemorySize=8g"

Bookie

bookiePort=3181
journalDirectory=/var/lib/apache-pulsar/data/bookkeeper/journal
minUsableSizeForIndexFileCreation=1073741824
allowLoopback=false
bookieDeathWatchInterval=1000
flushInterval=60000
useHostNameAsBookieID=false
gcWaitTime=900000
gcOverreplicatedLedgerWaitTime=86400000
numAddWorkerThreads=0
numReadWorkerThreads=32
numHighPriorityWorkerThreads=8
maxPendingReadRequestsPerThread=2500
maxPendingAddRequestsPerThread=10000
readWorkerThreadsThrottlingEnabled=true
enableBusyWait=false
auditorPeriodicBookieCheckInterval=86400
rereplicationEntryBatchSize=100
openLedgerRereplicationGracePeriod=30000
autoRecoveryDaemonEnabled=true
lostBookieRecoveryDelay=0
useV2WireProtocol=true
reppDnsResolverClass=org.apache.pulsar.zookeeper.ZkBookieRackAffinityMapping
serverTcpNoDelay=true
nettyMaxFrameSizeBytes=16787456
journalFormatVersionToWrite=5
journalMaxSizeMB=2048
journalMaxBackups=5
journalPreAllocSizeMB=16
journalWriteBufferSizeKB=64
journalRemoveFromPageCache=true
journalWriteData=true
journalSyncData=true
journalAdaptiveGroupWrites=true
journalMaxGroupWaitMSec=1
journalBufferedWritesThreshold=524288
numJournalCallbackThreads=8
journalAlignmentSize=4096
journalFlushWhenQueueEmpty=false
ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage
ledgerDirectories=/var/lib/apache-pulsar/data/bookkeeper/ledgers,/var/lib/apache-pulsar/data2/bookkeeper/ledgers,/var/lib/apache-pulsar/data3/bookkeeper/ledgers,/var/lib/apache-pulsar/data4/bookkeeper/ledgers
auditorPeriodicCheckInterval=604800
openFileLimit=0
fileInfoFormatVersionToWrite=0
pageLimit=0
zkLedgersRootPath=/core-apache-pulsar-bookkeeper
logSizeLimit=1073741824
entryLogFilePreallocationEnabled=true
flushEntrylogBytes=33554432
readBufferSizeBytes=4096
writeBufferSizeBytes=65536
compactionRate=1000
minorCompactionThreshold=0.2
minorCompactionInterval=3600
compactionMaxOutstandingRequests=100000
majorCompactionThreshold=0.5
majorCompactionInterval=86400
isThrottleByBytes=false
compactionRateByEntries=1000
compactionRateByBytes=1000000
enableStatistics=true
statsProviderClass=org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider
prometheusStatsHttpPort=8000
readOnlyModeEnabled=true
diskUsageThreshold=0.95
diskCheckInterval=10000
zkTimeout=30000
zkEnableSecurity=false
httpServerEnabled=false
httpServerPort=8000
httpServerClass=org.apache.bookkeeper.http.vertx.VertxHttpServer
dbStorage_readAheadCacheBatchSize=3000
dbStorage_rocksDB_blockCacheSize=
dbStorage_rocksDB_writeBufferSizeMB=64
dbStorage_rocksDB_sstSizeInMB=64
dbStorage_rocksDB_blockSize=65536
dbStorage_rocksDB_bloomFilterBitsPerKey=10
dbStorage_rocksDB_numLevels=-1
dbStorage_rocksDB_numFilesInLevel0=4
dbStorage_rocksDB_maxSizeInLevel1MB=256

Broker

brokerServicePort=6650
webServicePort=8080
bindAddress=0.0.0.0
advertisedListeners=external:pulsar://<%= @fqdn -%>:6650
haProxyProtocolEnabled=false
numOrderedExecutorThreads=8
numCacheExecutorThreadPoolSize=10
enableBusyWait=false
maxTenants=0
failureDomainsEnabled=false
metadataStoreSessionTimeoutMillis=30000
metadataStoreOperationTimeoutSeconds=30
metadataStoreCacheExpirySeconds=300
brokerShutdownTimeoutMs=60000
skipBrokerShutdownOnOOM=false
backlogQuotaCheckEnabled=true
backlogQuotaCheckIntervalInSeconds=60
backlogQuotaDefaultLimitBytes=-1
backlogQuotaDefaultLimitSecond=-1
backlogQuotaDefaultRetentionPolicy=producer_request_hold
ttlDurationDefaultInSeconds=0
allowAutoTopicCreation=true
allowAutoTopicCreationType=non-partitioned
allowAutoSubscriptionCreation=true
defaultNumPartitions=1
brokerDeleteInactiveTopicsEnabled=false
brokerDeleteInactiveTopicsFrequencySeconds=60
brokerDeleteInactiveTopicsMode=delete_when_no_subscriptions
brokerDeleteInactivePartitionedTopicMetadataEnabled=false
forceDeleteTenantAllowed=false
forceDeleteNamespaceAllowed=false
maxPendingPublishRequestsPerConnection=1000
messageExpiryCheckIntervalInMinutes=5
activeConsumerFailoverDelayTimeMillis=1000
subscriptionExpirationTimeMinutes=0
subscriptionRedeliveryTrackerEnabled=true
subscriptionExpiryCheckIntervalInMinutes=5
subscriptionTypesEnabled=Exclusive,Shared,Failover,Key_Shared
subscriptionKeySharedUseConsistentHashing=true
subscriptionKeySharedConsistentHashingReplicaPoints=100
subscriptionBacklogScanMaxTimeMs=120000
subscriptionBacklogScanMaxEntries=10000
brokerDeduplicationEnabled=false
brokerDeduplicationMaxNumberOfProducers=10000
brokerDeduplicationSnapshotFrequencyInSeconds=120
brokerDeduplicationSnapshotIntervalSeconds=120
brokerDeduplicationEntriesInterval=1000
brokerDeduplicationProducerInactivityTimeoutMinutes=360
defaultNumberOfNamespaceBundles=4
maxNamespacesPerTenant=0
maxTopicsPerNamespace=0
brokerMaxConnections=0
brokerMaxConnectionsPerIp=0
isAllowAutoUpdateSchemaEnabled=true
clientLibraryVersionCheckEnabled=false
preferLaterVersions=false
maxUnackedMessagesPerConsumer=50000
maxUnackedMessagesPerSubscription=200000
maxUnackedMessagesPerBroker=0
maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16
unblockStuckSubscriptionEnabled=false
topicPublisherThrottlingTickTimeMillis=10
preciseTopicPublishRateLimiterEnable=false
brokerPublisherThrottlingTickTimeMillis=50
brokerPublisherThrottlingMaxMessageRate=0
brokerPublisherThrottlingMaxByteRate=0
maxPublishRatePerTopicInMessages=0
maxPublishRatePerTopicInBytes=0
subscribeThrottlingRatePerConsumer=0
subscribeRatePeriodPerConsumerInSecond=30
dispatchThrottlingRateInMsg=0
dispatchThrottlingRateInByte=0
dispatchThrottlingRatePerTopicInMsg=0
dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnBatchMessageEnabled=false
dispatchThrottlingRatePerSubscriptionInMsg=0
dispatchThrottlingRatePerSubscriptionInByte=0
dispatchThrottlingRatePerReplicatorInMsg=0
dispatchThrottlingRatePerReplicatorInByte=0
dispatchThrottlingRateRelativeToPublishRate=false
dispatchThrottlingOnNonBacklogConsumerEnabled=true
dispatcherMaxReadBatchSize=100
dispatcherDispatchMessagesInSubscriptionThread=true
dispatcherMaxReadSizeBytes=16777216
dispatcherMinReadBatchSize=1
dispatcherMaxRoundRobinBatchSize=20
dispatcherReadFailureBackoffInitialTimeInMs=15000
dispatcherReadFailureBackoffMaxTimeInMs=60000
dispatcherReadFailureBackoffMandatoryStopTimeInMs=0
preciseDispatcherFlowControl=false
maxConcurrentLookupRequest=50000
maxConcurrentTopicLoadRequest=5000
maxConcurrentNonPersistentMessagePerConnection=1000
enablePersistentTopics=true
enableNonPersistentTopics=true
enableRunBookieTogether=false
enableRunBookieAutoRecoveryTogether=false
maxProducersPerTopic=0
maxSameAddressProducersPerTopic=0
encryptionRequireOnProducer=false
maxConsumersPerTopic=0
maxSameAddressConsumersPerTopic=0
maxSubscriptionsPerTopic=0
maxConsumersPerSubscription=0
maxMessageSize=16777216
brokerServiceCompactionMonitorIntervalInSeconds=60
brokerServiceCompactionThresholdInBytes=0
brokerServiceCompactionPhaseOneLoopTimeInSeconds=30
delayedDeliveryEnabled=true
delayedDeliveryTickTimeMillis=1000
isDelayedDeliveryDeliverAtTimeStrict=false
delayedDeliveryFixedDelayDetectionLookahead=50000
acknowledgmentAtBatchIndexLevelEnabled=false
enableReplicatedSubscriptions=true
replicatedSubscriptionsSnapshotFrequencyMillis=1000
replicatedSubscriptionsSnapshotTimeoutSeconds=30
replicatedSubscriptionsSnapshotMaxCachedPerSubscription=10
maxMessagePublishBufferSizeInMB=
retentionCheckIntervalInSeconds=120
maxMessageSizeCheckIntervalInSeconds=60
maxNumPartitionsPerPartitionedTopic=0
zookeeperSessionExpiredPolicy=reconnect
systemTopicEnabled=true
schemaRegistryCompatibilityCheckers=org.apache.pulsar.broker.service.schema.JsonSchemaCompatibilityCheck,org.apache.pulsar.broker.service.schema.AvroSchemaCompatibilityCheck,org.apache.pulsar.broker.service.schema.ProtobufNativeSchemaCompatibilityCheck
systemTopicSchemaCompatibilityStrategy=ALWAYS_COMPATIBLE
topicLevelPoliciesEnabled=true
topicFencingTimeoutSeconds=0
authenticateOriginalAuthData=false
tlsCertRefreshCheckDurationSec=300
tlsAllowInsecureConnection=false
tlsRequireTrustedClientCertOnConnect=false
webServiceTlsProvider=Conscrypt
tlsEnabledWithKeyStore=false
tlsKeyStoreType=JKS
tlsTrustStoreType=JKS
brokerClientTlsEnabled=false
brokerClientTlsEnabledWithKeyStore=false
brokerClientTlsKeyStoreType=JKS
brokerClientTlsTrustStoreType=JKS
metadataStoreBatchingEnabled=true
metadataStoreBatchingMaxDelayMillis=5
metadataStoreBatchingMaxOperations=1000
metadataStoreBatchingMaxSizeKb=128
authenticationEnabled=false
authenticationRefreshCheckSeconds=60
authorizationEnabled=false
authorizationProvider=org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider
authorizationAllowWildcardsMatching=false
saslJaasClientAllowedIds=.*pulsar.*
saslJaasServerSectionName=PulsarBroker
saslJaasServerRoleTokenSignerSecretPath=
httpMaxRequestSize=-1
disableHttpDebugMethods=false
httpRequestsLimitEnabled=false
httpRequestsMaxPerSecond=100.0
httpServerThreadPoolQueueSize=8192
httpServerAcceptQueueSize=8192
maxHttpServerConnections=2048
maxConcurrentHttpRequests=1024
bookkeeperClientTimeoutInSeconds=30
bookkeeperClientSeparatedIoThreadsEnabled=false
bookkeeperClientSpeculativeReadTimeoutInMillis=0
bookkeeperNumberOfChannelsPerBookie=16
bookkeeperUseV2WireProtocol=true
bookkeeperClientHealthCheckEnabled=true
bookkeeperClientHealthCheckIntervalSeconds=60
bookkeeperClientHealthCheckErrorThresholdPerInterval=5
bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800
bookkeeperClientQuarantineRatio=1.0
bookkeeperClientGetBookieInfoIntervalSeconds=86400
bookkeeperClientGetBookieInfoRetryIntervalSeconds=60
bookkeeperClientRackawarePolicyEnabled=true
bookkeeperClientRegionawarePolicyEnabled=false
bookkeeperClientMinNumRacksPerWriteQuorum=2
bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false
bookkeeperClientReorderReadSequenceEnabled=false
bookkeeperEnableStickyReads=true
bookkeeperTLSProviderFactoryClass=org.apache.bookkeeper.tls.TLSContextFactory
bookkeeperTLSClientAuthentication=false
bookkeeperTLSKeyFileType=PEM
bookkeeperTLSTrustCertTypes=PEM
bookkeeperTlsCertFilesRefreshDurationSeconds=300
bookkeeperDiskWeightBasedPlacementEnabled=false
bookkeeperExplicitLacIntervalInMills=0
bookkeeperClientExposeStatsToPrometheus=true
managedLedgerDefaultEnsembleSize=1
managedLedgerDefaultWriteQuorum=1
managedLedgerDefaultAckQuorum=1
managedLedgerCursorPositionFlushSeconds=60
managedLedgerStatsPeriodSeconds=60
managedLedgerDigestType=CRC32C
managedLedgerCacheCopyEntries=false
managedLedgerCacheEvictionWatermark=0.9
managedLedgerCacheEvictionIntervalMs=10
managedLedgerCacheEvictionTimeThresholdMillis=1000
managedLedgerCursorBackloggedThreshold=1000
managedLedgerDefaultMarkDeleteRateLimit=1.0
managedLedgerMaxEntriesPerLedger=50000
managedLedgerMinLedgerRolloverTimeMinutes=10
managedLedgerMaxLedgerRolloverTimeMinutes=240
managedLedgerInactiveLedgerRolloverTimeSeconds=0
managedLedgerMaxSizePerLedgerMbytes=2048
managedLedgerOffloadDeletionLagMs=14400000
managedLedgerOffloadAutoTriggerSizeThresholdBytes=-1
managedLedgerCursorMaxEntriesPerLedger=50000
managedLedgerCursorRolloverTimeInSeconds=14400
managedLedgerMaxUnackedRangesToPersist=10000
managedLedgerMaxReadsInFlightSizeInMB=0
managedLedgerMaxUnackedRangesToPersistInMetadataStore=1000
autoSkipNonRecoverableData=false
lazyCursorRecovery=false
managedLedgerMetadataOperationsTimeoutSeconds=60
managedLedgerReadEntryTimeoutSeconds=0
managedLedgerAddEntryTimeoutSeconds=0
managedLedgerPrometheusStatsLatencyRolloverSeconds=60
managedLedgerTraceTaskExecution=true
managedLedgerNewEntriesCheckDelayInMillis=10
loadBalancerEnabled=true
loadBalancerReportUpdateThresholdPercentage=10
loadBalancerReportUpdateMinIntervalMillis=5000
loadBalancerReportUpdateMaxIntervalMinutes=15
loadBalancerHostUsageCheckIntervalMinutes=1
loadBalancerSheddingEnabled=true
loadBalancerSheddingIntervalMinutes=1
loadBalancerSheddingGracePeriodMinutes=30
loadBalancerBrokerMaxTopics=50000
loadBalancerBrokerOverloadedThresholdPercentage=85
loadBalancerResourceQuotaUpdateIntervalMinutes=15
loadBalancerAutoBundleSplitEnabled=true
loadBalancerAutoUnloadSplitBundlesEnabled=true
loadBalancerNamespaceBundleMaxTopics=1000
loadBalancerNamespaceBundleMaxSessions=1000
loadBalancerNamespaceBundleMaxMsgRate=30000
loadBalancerNamespaceBundleMaxBandwidthMbytes=100
loadBalancerNamespaceMaximumBundles=128
loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl
supportedNamespaceBundleSplitAlgorithms=range_equally_divide,topic_count_equally_divide,specified_positions_divide
defaultNamespaceBundleSplitAlgorithm=range_equally_divide
loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.impl.ThresholdShedder
loadBalancerLoadPlacementStrategy=org.apache.pulsar.broker.loadbalance.impl.LeastLongTermMessageRate
loadBalancerBrokerThresholdShedderPercentage=10
loadBalancerAverageResourceUsageDifferenceThresholdPercentage=10
loadBalancerMsgRateDifferenceShedderThreshold=50
loadBalancerMsgThroughputMultiplierDifferenceShedderThreshold=4
loadBalancerHistoryResourcePercentage=0.9
loadBalancerBandwithInResourceWeight=1.0
loadBalancerBandwithOutResourceWeight=1.0
loadBalancerCPUResourceWeight=1.0
loadBalancerMemoryResourceWeight=1.0
loadBalancerDirectMemoryResourceWeight=1.0
loadBalancerBundleUnloadMinThroughputThreshold=10
namespaceBundleUnloadingTimeoutMs=60000
replicationMetricsEnabled=true
replicationConnectionsPerBroker=16
replicationProducerQueueSize=1000
replicatorPrefix=pulsar.repl
replicationPolicyCheckDurationSeconds=600
defaultRetentionTimeInMinutes=0
defaultRetentionSizeInMB=0
keepAliveIntervalSeconds=30
webSocketServiceEnabled=false
webSocketSessionIdleTimeoutMillis=300000
webSocketMaxTextFrameSize=1048576
authenticateMetricsEndpoint=false
exposeTopicLevelMetricsInPrometheus=true
exposeConsumerLevelMetricsInPrometheus=true
metricsBufferResponse=false
exposeProducerLevelMetricsInPrometheus=false
exposeManagedLedgerMetricsInPrometheus=true
exposeManagedCursorMetricsInPrometheus=false
metricsServletTimeoutMs=30000
exposeBundlesMetricsInPrometheus=false
functionsWorkerEnabled=false
functionsWorkerEnablePackageManagement=false
exposePublisherStats=true
statsUpdateFrequencyInSecs=60
statsUpdateInitialDelayInSecs=60
exposePreciseBacklogInPrometheus=false
splitTopicAndPartitionLabelInPrometheus=false
aggregatePublisherStatsByProducerName=false
schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory
isSchemaValidationEnforced=false
schemaCompatibilityStrategy=FULL
offloadersDirectory=./offloaders
managedLedgerOffloadMaxThreads=2
managedLedgerOffloadPrefetchRounds=1
managedLedgerUnackedRangesOpenCacheSetEnabled=true
s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864
gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576
fileSystemProfilePath=../conf/filesystem_offload_core_site.xml
transactionCoordinatorEnabled=false
transactionMetadataStoreProviderClassName=org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider
transactionBufferSnapshotMaxTransactionCount=1000
transactionBufferSnapshotMinTimeInMillis=5000
transactionBufferClientMaxConcurrentRequests=1000
maxActiveTransactionsPerCoordinator=0
transactionPendingAckLogIndexMinLag=500
transactionBufferClientOperationTimeoutInMills=3000
transactionLogBatchedWriteEnabled=false
transactionLogBatchedWriteMaxRecords=512
transactionLogBatchedWriteMaxSize=4194304
transactionLogBatchedWriteMaxDelayInMillis=1
transactionPendingAckBatchedWriteEnabled=false
transactionPendingAckBatchedWriteMaxRecords=512
transactionPendingAckBatchedWriteMaxSize=4194304
transactionPendingAckBatchedWriteMaxDelayInMillis=1
enablePackagesManagement=false
packagesManagementStorageProvider=org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider
packagesReplicas=1
packagesManagementLedgerRootPath=/ledgers
strictBookieAffinityEnabled=false
zooKeeperSessionTimeoutMillis=-1
zooKeeperOperationTimeoutSeconds=-1
zooKeeperCacheExpirySeconds=-1
replicationTlsEnabled=false
brokerServicePurgeInactiveFrequencyInSeconds=60
backlogQuotaDefaultLimitGB=-1
tlsEnabled=false
subscriptionKeySharedEnable=true
managedLedgerMaxUnackedRangesToPersistInZooKeeper=-1
persistentUnackedRangesWithMultipleEntriesEnabled=false
managedLedgerCacheEvictionFrequency=0