Skip to content

Commit ab3ca3f

Browse files
dfa1claude
andcommitted
perf(bitpacked): hoist per-row bookkeeping out of unpackLoop64 block loop
Inner row body computed seven row-dependent quantities (currWord, nextWord, shift, remainingBits, currentBits, loMask, hiMask, plus FL_ORDER lookup) from `row` and `bitWidth` on every iteration. `bitWidth` is constant for the duration of unpackLoop64, so these only need to be computed once. For a 10M-row I64 column at bitWidth ~20, this drops 7 × 64 × ~9750 blocks ≈ 4.4M ops out of the hot path. JFR shows BitpackedEncoding$Decoder.unpackLoop64 as the largest visible Java frame (31% of RUNNABLE time on RustVsJavaReadBenchmark.javaReadVolume). Pre-compute eight 64-entry int[]/long[] tables at method entry, look up per row inside the block loop. Inner lane loop body is unchanged (buf.get / >>> / & / out.set). Bench (M5, JDK 25, 5 warmup × 3s + 10 measurement × 5s, fork 1): RustVsJavaReadBenchmark.javaReadVolume before: 109.319 ± 1.042 ops/s after: 113.526 ± 1.033 ops/s (+3.8 %) Error bars don't overlap so the gain is real, just modest. Vector API rewrite of the inner lane loop (TODO line ~) would be the next material step; this is a no-risk micro-cleanup pending that work. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 77e59ad commit ab3ca3f

1 file changed

Lines changed: 51 additions & 25 deletions

File tree

core/src/main/java/io/github/dfa1/vortex/encoding/BitpackedEncoding.java

Lines changed: 51 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -537,32 +537,60 @@ private static void unpackLoop64(MemorySegment buf, int bitWidth, int offset, lo
537537
int blockCount = (int) ((totalElems + 1023) / 1024);
538538
long bitMask = bitWidth == 64 ? -1L : (1L << bitWidth) - 1L;
539539

540+
// Per-row bookkeeping depends only on `row` and `bitWidth`; both are constant for
541+
// the duration of this call. Pre-compute once instead of recomputing 7 derived
542+
// ints per row per block. For a 10M-row I64 column with bitWidth ~20 that's
543+
// 65K rows × 7 ops eliminated from the inner loop — material on a kernel that
544+
// shows up as the largest visible Java frame in JFR.
545+
int[] shifts = new int[64];
546+
int[] remainingBits = new int[64];
547+
int[] currentBits = new int[64];
548+
long[] loMasks = new long[64];
549+
long[] hiMasks = new long[64];
550+
long[] currWordByteBase = new long[64]; // lanes * currWord * 8
551+
long[] nextWordByteBase = new long[64]; // lanes * nextWord * 8 (0 when remainingBits == 0)
552+
// Output offset within a block, * 8 (bytes). Per-row, independent of block.
553+
long[] outRowByteOff = new long[64];
554+
for (int row = 0; row < 64; row++) {
555+
int currWord = (row * bitWidth) / 64;
556+
int nextWord = ((row + 1) * bitWidth) / 64;
557+
shifts[row] = (row * bitWidth) % 64;
558+
int rem = (nextWord > currWord) ? ((row + 1) * bitWidth) % 64 : 0;
559+
remainingBits[row] = rem;
560+
int curr = bitWidth - rem;
561+
currentBits[row] = curr;
562+
loMasks[row] = rem > 0 ? (1L << curr) - 1L : 0L;
563+
hiMasks[row] = rem > 0 ? (1L << rem) - 1L : 0L;
564+
currWordByteBase[row] = (long) lanes * currWord * 8L;
565+
nextWordByteBase[row] = rem > 0 ? (long) lanes * nextWord * 8L : 0L;
566+
int o = row / 8;
567+
int s = row % 8;
568+
outRowByteOff[row] = (long) (FL_ORDER[o] * 16 + s * 128) * 8L;
569+
}
570+
540571
long blockByteOff = 0L;
541572
long blockByteStride = 128L * bitWidth;
542573
for (int block = 0; block < blockCount; block++, blockByteOff += blockByteStride) {
543574
int blockLogicStart = block * 1024 - offset;
544575
boolean fullBlock = blockLogicStart >= 0 && (long) blockLogicStart + 1023L < rowCount;
576+
long blockOutByteBase = (long) blockLogicStart * 8L;
545577

546578
if (fullBlock) {
547579
for (int row = 0; row < 64; row++) {
548-
int currWord = (row * bitWidth) / 64;
549-
int nextWord = ((row + 1) * bitWidth) / 64;
550-
int shift = (row * bitWidth) % 64;
551-
int remainingBits = (nextWord > currWord) ? ((row + 1) * bitWidth) % 64 : 0;
552-
int currentBits = bitWidth - remainingBits;
553-
int o = row / 8;
554-
int s = row % 8;
555-
long outBase = (long) (blockLogicStart + FL_ORDER[o] * 16 + s * 128) * 8;
556-
long wordBase = blockByteOff + (long) lanes * currWord * 8;
557-
if (remainingBits > 0) {
558-
long hiBase = blockByteOff + (long) lanes * nextWord * 8;
559-
long loMask = (1L << currentBits) - 1L;
560-
long hiMask = (1L << remainingBits) - 1L;
580+
int shift = shifts[row];
581+
int rem = remainingBits[row];
582+
int curr = currentBits[row];
583+
long outBase = blockOutByteBase + outRowByteOff[row];
584+
long wordBase = blockByteOff + currWordByteBase[row];
585+
if (rem > 0) {
586+
long hiBase = blockByteOff + nextWordByteBase[row];
587+
long loMask = loMasks[row];
588+
long hiMask = hiMasks[row];
561589
long laneOff = 0L;
562590
for (int lane = 0; lane < lanes; lane++, laneOff += 8L) {
563591
long lo = (buf.get(PTypeIO.LE_LONG, wordBase + laneOff) >>> shift) & loMask;
564592
long hi = buf.get(PTypeIO.LE_LONG, hiBase + laneOff) & hiMask;
565-
out.set(PTypeIO.LE_LONG, outBase + laneOff, lo | (hi << currentBits));
593+
out.set(PTypeIO.LE_LONG, outBase + laneOff, lo | (hi << curr));
566594
}
567595
} else {
568596
long laneOff = 0L;
@@ -574,29 +602,27 @@ private static void unpackLoop64(MemorySegment buf, int bitWidth, int offset, lo
574602
}
575603
} else {
576604
for (int row = 0; row < 64; row++) {
577-
int currWord = (row * bitWidth) / 64;
578-
int nextWord = ((row + 1) * bitWidth) / 64;
579-
int shift = (row * bitWidth) % 64;
580-
int remainingBits = (nextWord > currWord) ? ((row + 1) * bitWidth) % 64 : 0;
581-
int currentBits = bitWidth - remainingBits;
605+
int shift = shifts[row];
606+
int rem = remainingBits[row];
607+
int curr = currentBits[row];
582608
int o = row / 8;
583609
int s = row % 8;
584610
int baseIdx = blockLogicStart + FL_ORDER[o] * 16 + s * 128;
585-
long wordBase = blockByteOff + (long) lanes * currWord * 8;
586-
long hiBase = (remainingBits > 0) ? blockByteOff + (long) lanes * nextWord * 8 : 0L;
587-
long loMask = (remainingBits > 0) ? (1L << currentBits) - 1L : 0L;
588-
long hiMask = (remainingBits > 0) ? (1L << remainingBits) - 1L : 0L;
611+
long wordBase = blockByteOff + currWordByteBase[row];
612+
long hiBase = rem > 0 ? blockByteOff + nextWordByteBase[row] : 0L;
613+
long loMask = loMasks[row];
614+
long hiMask = hiMasks[row];
589615
for (int lane = 0; lane < lanes; lane++) {
590616
int logicalIdx = baseIdx + lane;
591617
if (logicalIdx < 0 || logicalIdx >= rowCount) {
592618
continue;
593619
}
594620
long src = buf.get(PTypeIO.LE_LONG, wordBase + (long) lane * 8);
595621
long value;
596-
if (remainingBits > 0) {
622+
if (rem > 0) {
597623
long lo = (src >>> shift) & loMask;
598624
long hi = buf.get(PTypeIO.LE_LONG, hiBase + (long) lane * 8) & hiMask;
599-
value = lo | (hi << currentBits);
625+
value = lo | (hi << curr);
600626
} else {
601627
value = (src >>> shift) & bitMask;
602628
}

0 commit comments

Comments
 (0)