Benchmark and performance

.gitignore

@@ -58,3 +58,4 @@ data
 wikidata
 qendpoint-store/wdbench-indexes
 wdbench-results
+wdbench-indexes

qendpoint-backend/pom.xml

@@ -43,7 +43,7 @@
         <json_simple.version>1.1.1</json_simple.version>
         <junit.version>4.13.2</junit.version>
         <lwjgl.version>3.3.1</lwjgl.version>
-        <rdf4j.version>5.0.0-SNAPSHOT</rdf4j.version>
+        <rdf4j.version>5.0.1</rdf4j.version>
         <spring.version>3.0.2</spring.version>
         <logback.version>1.4.5</logback.version>
 

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/compact/sequence/SequenceLog64Map.java

@@ -53,7 +53,10 @@
  */
 public class SequenceLog64Map implements Sequence, Closeable {
 	private static final byte W = 64;
-	private static final long LONGS_PER_BUFFER = 128 * 1024 * 1024; // 128*8 =
+	public static final int W_LEFT_SHIFT = (W << 1);
+	private static final int LONGS_PER_BUFFER = 128 * 1024 * 1024; // 128*8 =
+	private static final int LOG2_LONGS_PER_BUFFER = Long.numberOfTrailingZeros(LONGS_PER_BUFFER);
+
 	// 1Gb per
 	// chunk.
 	private CloseMappedByteBuffer[] buffers;
@@ -62,6 +65,8 @@ public class SequenceLog64Map implements Sequence, Closeable {
 	private final long numentries;
 	private long lastword;
 	private final long numwords;
+	private final int W_numbits;
+	private final int W_LEFT_SHIFT_MINUS_NUMBITS;
 
 	public SequenceLog64Map(File f) throws IOException {
 		// Read from the beginning of the file
@@ -80,6 +85,8 @@ private SequenceLog64Map(CountInputStream in, File f, boolean closeInput) throws
 			throw new IllegalFormatException("Trying to read a LogArray but the data is not LogArray");
 		}
 		numbits = crcin.read();
+		W_numbits = W - numbits;
+		W_LEFT_SHIFT_MINUS_NUMBITS = W_LEFT_SHIFT - numbits;
 		numentries = VByte.decode(crcin);
 
 		if (!crcin.readCRCAndCheck()) {
@@ -111,6 +118,8 @@ private SequenceLog64Map(CountInputStream in, File f, boolean closeInput) throws
 
 	public SequenceLog64Map(int numbits, long numentries, File f) throws IOException {
 		this.numbits = numbits;
+		this.W_numbits = W - numbits;
+		this.W_LEFT_SHIFT_MINUS_NUMBITS = W_LEFT_SHIFT - numbits;
 		this.numentries = numentries;
 		this.numwords = SequenceLog64.numWordsFor(numbits, numentries);
 
@@ -178,32 +187,41 @@ private long getWord(long w) {
 			return lastword;
 		}
 
-		return buffers[(int) (w / LONGS_PER_BUFFER)].getLong((int) ((w % LONGS_PER_BUFFER) * 8));
+		return buffers[(int) (w >> LOG2_LONGS_PER_BUFFER)].getLong((int) ((w & (LONGS_PER_BUFFER - 1)) << 3));
+//		return buffers[(int) (w / LONGS_PER_BUFFER)].getLong((int) ((w % LONGS_PER_BUFFER) * 8));
 	}
 
 	/*
 	 * (non-Javadoc)
 	 * @see hdt.triples.array.Stream#get(long)
 	 */
+
 	@Override
 	public long get(long index) {
 		if (index < 0 || index >= numentries) {
 			throw new IndexOutOfBoundsException(index + " < 0 || " + index + ">= " + numentries);
 		}
-		if (numbits == 0)
+		if (numbits == 0) {
 			return 0;
+		}
 
 		long bitPos = index * numbits;
-		long i = bitPos / W;
 		int j = (int) (bitPos % W);
-		long result;
 		if (j + numbits <= W) {
-			result = (getWord(i) << (W - j - numbits)) >>> (W - numbits);
+			return extracted1(bitPos, j);
 		} else {
-			result = getWord(i) >>> j;
-			result = result | (getWord(i + 1) << ((W << 1) - j - numbits)) >>> (W - numbits);
+			return extracted(bitPos, j);
 		}
-		return result;
+	}
+
+	private long extracted(long bitPos, int j) {
+		long i = bitPos / W;
+		return getWord(i) >>> j | (getWord(i + 1) << (W_LEFT_SHIFT_MINUS_NUMBITS - j)) >>> W_numbits;
+	}
+
+	private long extracted1(long bitPos, int j) {
+		long i = bitPos / W;
+		return (getWord(i) << (W_numbits - j)) >>> (W_numbits);
 	}
 
 	/*

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/enums/TripleComponentOrder.java

@@ -20,6 +20,7 @@
 package com.the_qa_company.qendpoint.core.enums;
 
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
@@ -98,6 +99,8 @@ public static <T, Z extends TripleComponentOrder> List<Z> fetchAllBestForCfg(int
 		return ret;
 	}
 
+	public static TripleComponentOrder preference;
+
 	/**
 	 * Search for an acceptable value in a map of orders
 	 *
@@ -107,12 +110,43 @@ public static <T, Z extends TripleComponentOrder> List<Z> fetchAllBestForCfg(int
 	 * @return find value, null for no matching value
 	 */
 	public static <T> T fetchBestForCfg(int flags, Map<? extends TripleComponentOrder, T> map) {
+
+		var tripleComponentOrders = EnumSet.noneOf(TripleComponentOrder.class);
+
 		for (Map.Entry<? extends TripleComponentOrder, T> e : map.entrySet()) {
 			if ((e.getKey().mask & flags) != 0) {
-				return e.getValue();
+				tripleComponentOrders.add(e.getKey());
+			}
+		}
+
+		if (tripleComponentOrders.isEmpty()) {
+			return null;
+		}
+
+		if (preference != null) {
+			if (tripleComponentOrders.contains(preference)) {
+				return map.get(preference);
 			}
+			throw new IllegalStateException("Preference not found in the list of acceptable orders");
 		}
-		return null;
+
+		if (tripleComponentOrders.contains(SOP)) {
+			return map.get(SOP);
+		}
+		if (tripleComponentOrders.contains(OPS)) {
+			return map.get(OPS);
+		}
+		if (tripleComponentOrders.contains(OSP)) {
+			return map.get(OSP);
+		}
+		if (tripleComponentOrders.contains(POS)) {
+			return map.get(POS);
+		}
+		if (tripleComponentOrders.contains(PSO)) {
+			return map.get(PSO);
+		}
+
+		return map.get(tripleComponentOrders.iterator().next());
 	}
 
 	/**

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriples.java

@@ -93,6 +93,7 @@
  */
 public class BitmapTriples implements TriplesPrivate, BitmapTriplesIndex {
 	private static final Logger log = LoggerFactory.getLogger(BitmapTriples.class);
+	public static boolean useDefaultOrder = true;
 
 	protected TripleComponentOrder order;
 
@@ -317,7 +318,7 @@ public SuppliableIteratorTripleID search(TripleID pattern, int searchMask) {
 		TripleOrderConvert.swapComponentOrder(reorderedPat, TripleComponentOrder.SPO, order);
 		int flags = reorderedPat.getPatternOrderFlags();
 
-		if ((flags & searchMask & this.order.mask) != 0) {
+		if (useDefaultOrder && (flags & searchMask & this.order.mask) != 0) {
 			// we can use the default order, so we use it
 			return new BitmapTriplesIterator(this, pattern);
 		}
@@ -1340,7 +1341,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
 			try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
 				// load from the path...
 
-				BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
+				BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel);
 				BitmapTriplesIndex old = indexes.put(order, idx);
 				indexesMask |= idx.getOrder().mask;
 				if (old != null) {
@@ -1357,7 +1358,7 @@ public void syncOtherIndexes(Path fileLocation, HDTOptions spec, ProgressListene
 				BitmapTriplesIndexFile.generateIndex(this, subIndexPath, order, spec, mListener);
 				try (FileChannel channel = FileChannel.open(subIndexPath, StandardOpenOption.READ)) {
 					// load from the path...
-					BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel, this);
+					BitmapTriplesIndex idx = BitmapTriplesIndexFile.map(subIndexPath, channel);
 					BitmapTriplesIndex old = indexes.put(order, idx);
 					indexesMask |= order.mask;
 					if (old != null) {

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/BitmapTriplesIndexFile.java

@@ -11,7 +11,6 @@
 import com.the_qa_company.qendpoint.core.compact.sequence.SequenceLog64BigDisk;
 import com.the_qa_company.qendpoint.core.enums.TripleComponentOrder;
 import com.the_qa_company.qendpoint.core.exceptions.IllegalFormatException;
-import com.the_qa_company.qendpoint.core.exceptions.SignatureIOException;
 import com.the_qa_company.qendpoint.core.iterator.utils.AsyncIteratorFetcher;
 import com.the_qa_company.qendpoint.core.iterator.utils.ExceptionIterator;
 import com.the_qa_company.qendpoint.core.iterator.utils.MapIterator;
@@ -34,7 +33,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InterruptedIOException;
-import java.nio.ByteOrder;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
@@ -61,49 +59,30 @@ public static Path getIndexPath(Path hdt, TripleComponentOrder order) {
 		return hdt.resolveSibling(hdt.getFileName() + "." + order.name().toLowerCase() + ".idx");
 	}
 
-	/**
-	 * Compute triples signature
-	 *
-	 * @param triples triples
-	 * @return signature
-	 */
-	public static long signature(BitmapTriples triples) {
-		return 0x484454802020L ^ triples.getNumberOfElements();
-	}
-
-	public static final byte[] MAGIC = "$HDTIDX1".getBytes(StandardCharsets.US_ASCII);
+	public static final byte[] MAGIC = "$HDTIDX0".getBytes(StandardCharsets.US_ASCII);
 
 	/**
 	 * Map a file from a file
 	 *
 	 * @param file    file
 	 * @param channel channel
-	 * @param triples triples
 	 * @return index
 	 * @throws IOException io
 	 */
-	public static BitmapTriplesIndex map(Path file, FileChannel channel, BitmapTriples triples) throws IOException {
+	public static BitmapTriplesIndex map(Path file, FileChannel channel) throws IOException {
 		try (CloseMappedByteBuffer header = IOUtil.mapChannel(file, channel, FileChannel.MapMode.READ_ONLY, 0,
-				MAGIC.length + 8)) {
+				MAGIC.length)) {
 			byte[] magicRead = new byte[MAGIC.length];
 
 			header.get(magicRead);
 
 			if (!Arrays.equals(magicRead, MAGIC)) {
 				throw new IOException(format("Can't read %s magic", file));
 			}
-
-			long signature = header.order(ByteOrder.LITTLE_ENDIAN).getLong(magicRead.length);
-
-			long currentSignature = signature(triples);
-			if (signature != currentSignature) {
-				throw new SignatureIOException(
-						format("Wrong signature for file 0x%x != 0x%x", signature, currentSignature));
-			}
 		}
 
 		CountInputStream stream = new CountInputStream(new BufferedInputStream(Channels.newInputStream(channel)));
-		stream.skipNBytes(MAGIC.length + 8);
+		stream.skipNBytes(MAGIC.length);
 
 		String orderCfg = IOUtil.readSizedString(stream, ProgressListener.ignore());
 
@@ -289,7 +268,6 @@ public static void generateIndex(BitmapTriples triples, Path destination, Triple
 				// saving the index
 				try (BufferedOutputStream output = new BufferedOutputStream(Files.newOutputStream(destination))) {
 					output.write(MAGIC);
-					IOUtil.writeLong(output, signature(triples));
 
 					IOUtil.writeSizedString(output, order.name(), listener);
 

qendpoint-core/src/main/java/com/the_qa_company/qendpoint/core/triples/impl/TripleOrderConvert.java

@@ -67,24 +67,33 @@ public static void swapComponentOrder(TripleID triple, TripleComponentOrder from
 		if (from == TripleComponentOrder.Unknown || to == TripleComponentOrder.Unknown) {
 			throw new IllegalArgumentException("Cannot swap Unknown Orders");
 		}
-		boolean swap1 = swap1tab[from.ordinal() - 1][to.ordinal() - 1];
-		boolean swap2 = swap2tab[from.ordinal() - 1][to.ordinal() - 1];
-		boolean swap3 = swap3tab[from.ordinal() - 1][to.ordinal() - 1];
 
-		if (swap1) {
-			long tmp = triple.getSubject();
-			triple.setSubject(triple.getPredicate());
-			triple.setPredicate(tmp);
+		swap1(triple, from, to);
+		swap2(triple, from, to);
+		swap3(triple, from, to);
+	}
+
+	private static void swap3(TripleID triple, TripleComponentOrder from, TripleComponentOrder to) {
+		if (swap3tab[from.ordinal() - 1][to.ordinal() - 1]) {
+			long tmp = triple.getPredicate();
+			triple.setPredicate(triple.getObject());
+			triple.setObject(tmp);
 		}
-		if (swap2) {
+	}
+
+	private static void swap2(TripleID triple, TripleComponentOrder from, TripleComponentOrder to) {
+		if (swap2tab[from.ordinal() - 1][to.ordinal() - 1]) {
 			long tmp = triple.getSubject();
 			triple.setSubject(triple.getObject());
 			triple.setObject(tmp);
 		}
-		if (swap3) {
-			long tmp = triple.getPredicate();
-			triple.setPredicate(triple.getObject());
-			triple.setObject(tmp);
+	}
+
+	private static void swap1(TripleID triple, TripleComponentOrder from, TripleComponentOrder to) {
+		if (swap1tab[from.ordinal() - 1][to.ordinal() - 1]) {
+			long tmp = triple.getSubject();
+			triple.setSubject(triple.getPredicate());
+			triple.setPredicate(tmp);
 		}
 	}
 }

qendpoint-store/pom.xml

@@ -41,7 +41,7 @@
         <json_simple.version>1.1.1</json_simple.version>
         <junit.version>4.13.2</junit.version>
         <lwjgl.version>3.3.1</lwjgl.version>
-        <rdf4j.version>5.0.0-SNAPSHOT</rdf4j.version>
+        <rdf4j.version>5.0.1</rdf4j.version>
         <logback.version>1.4.5</logback.version>
 
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -173,6 +173,18 @@
             <artifactId>qendpoint-core</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>1.37</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>1.37</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

qendpoint-store/src/main/java/com/the_qa_company/qendpoint/model/EndpointStoreValueFactory.java

@@ -99,7 +99,7 @@ public IRI createIRI(String iri) {
 			}
 		}
 		if (id != -1) {
-			return new SimpleIRIHDT(hdt, position, id);
+			return new SimpleIRIHDT(hdt.getDictionary(), position, id);
 		} else {
 			return super.createIRI(iri);
 		}