Set up CI with Azure Pipelines and Update Readme

Author: zqxjjj

.github/workflows/codeql.yml

@@ -0,0 +1,78 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '16 4 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    #- name: Autobuild
+    #  uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+     - name: Build
+     - run: |
+         echo "Run, Build Application using script"
+         git submodule update --init --recursive
+         ./scripts/patch.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker-image.yml

@@ -0,0 +1,21 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build the Docker image
+      run: |
+      git submodule update --init --recursive
+      ./scripts/patch.sh
+      docker build . --file Dockerfile --tag MSVBASE:$(date +%s)

CMakeLists.txt

@@ -1,3 +1,6 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
 cmake_minimum_required(VERSION 3.14.0)
 project(vectordb VERSION 0.1.0)
 
@@ -84,6 +87,8 @@ add_library(${PROJECT_NAME} SHARED
     src/model_mng.cpp
     src/topk.cpp
     src/multicol_topk.cpp
+    src/spannindex.cpp
+    src/spannindex_scan.cpp
 )
 
 target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_14)
@@ -111,28 +116,29 @@ endif(WIN32)
 
 ## Find PostgreSQL Paths
 
-foreach(suffix ${PostgreSQL_KNOWN_VERSIONS})
-  if(WIN32)
-    list(APPEND PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES
-        "PostgreSQL/${suffix}/bin")
-  endif()
-  if(UNIX)
-    list(APPEND PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES
-        "postgresql${suffix}"
-        "postgresql/${suffix}"
-        "pgsql-${suffix}/bin")
-  endif()
-endforeach()
-find_program(PostgreSQL_PG_CONFIG pg_config
-NAMES ${ARGN}
-PATHS
-  ${PostgreSQL_ROOT_DIRECTORIES}
-PATH_SUFFIXES
-  bin
-  ${PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES}
+#foreach(suffix ${PostgreSQL_KNOWN_VERSIONS})
+#  if(WIN32)
+#    list(APPEND PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES
+#        "PostgreSQL/${suffix}/bin")
+#  endif()
+#  if(UNIX)
+#    list(APPEND PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES
+#        "postgresql${suffix}"
+#        "postgresql/${suffix}"
+#        "pgsql-${suffix}/bin")
+#  endif()
+#endforeach()
+#find_program(PostgreSQL_PG_CONFIG pg_config
+#NAMES ${ARGN}
+#PATHS
+#  ${PostgreSQL_ROOT_DIRECTORIES}
+#PATH_SUFFIXES
+#  bin
+#  ${PostgreSQL_BINARY_ADDITIONAL_SEARCH_SUFFIXES}
 # Help the user find it if we cannot.
-DOC "Set the PostgreSQL_BINARY_DIR cmake cache entry to the top-level directory containing the PostgreSQL binaries."
-)
+#DOC "Set the PostgreSQL_BINARY_DIR cmake cache entry to the top-level directory containing the PostgreSQL binaries."
+#)
+set(PostgreSQL_PG_CONFIG  /u01/app/postgres/product/13.4/bin/pg_config)
 if (NOT PostgreSQL_PG_CONFIG)
     message(FATAL_ERROR "Unable to find 'pg_config'")
 endif ()

Dockerfile

@@ -1,5 +1,5 @@
-FROM debian:bullseye-slim
-
+#FROM debian:bullseye-slim
+FROM gcc:12.3.0
 # make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
 ENV LANG en_US.utf8
 ENV PG_MAJOR 13
@@ -24,7 +24,7 @@ RUN set -ex \
     procps \
     sysstat \
     libldap2-dev \
-    python-dev \
+    python3-dev \
     libreadline-dev \
     libssl-dev \
     bison \
@@ -37,49 +37,49 @@ RUN set -ex \
     make \
     gcc \
     unzip \
-    python \
+    python3 \
     locales \
     wget \
     \
-    && rm -rf /var/lib/apt/lists/* \
+  #  && rm -rf /var/lib/apt/lists/* \
     && localedef -i en_US -c -f UTF-8 en_US.UTF-8
 
 RUN apt-get update && \
-	apt-get install -y --no-install-recommends wget git golang-go python-dev swig vim\
+	apt-get install -y --no-install-recommends wget git golang-go  swig vim\
     libboost-filesystem-dev libboost-test-dev libboost-serialization-dev libboost-regex-dev libboost-serialization-dev libboost-regex-dev libboost-thread-dev libboost-system-dev
 
-RUN wget "https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz" --no-check-certificate -q -O - \
+RUN wget "https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.gz" --no-check-certificate -q -O - \
         | tar -xz && \
-        cd boost_1_71_0 && \
+        cd boost_1_81_0 && \
         ./bootstrap.sh && \
         ./b2 install && \
         ldconfig && \
-        cd .. && rm -rf boost_1_71_0
+        cd .. && rm -rf boost_1_81_0
 
 RUN wget "https://github.com/Kitware/CMake/releases/download/v3.14.4/cmake-3.14.4-Linux-x86_64.tar.gz" --no-check-certificate -q -O - \
 	| tar -xz --strip-components=1 -C /usr/local
 
-RUN apt-get install -y software-properties-common && add-apt-repository 'deb http://archive.debian.org/debian stretch stretch-security main contrib non-free' && apt-get update && apt-get install -y openjdk-8-jdk
+#RUN apt-get install -y software-properties-common && add-apt-repository 'deb http://archive.debian.org/debian stretch stretch-security main contrib non-free' && apt-get update && apt-get install -y openjdk-8-jdk
 
-RUN apt install -y pip && pip install numpy && pip install pandas
+#RUN apt install -y pip && pip install numpy && pip install pandas
 
 RUN git config --global http.sslverify false
 
-RUN wget http://ftp.gnu.org/gnu/gcc/gcc-12.2.0/gcc-12.2.0.tar.gz \
-    && tar -zxvf gcc-12.2.0.tar.gz \
-    && cd gcc-12.2.0 \
-    && ./contrib/download_prerequisites \
-    && mkdir build \
-    && cd build/ \
-    && ../configure -enable-checking=release -enable-languages=c,c++ -disable-multilib \
-    && make -j$(nproc) \
-    && make install \
-    && rm /usr/bin/gcc \
-    && ln -s /usr/local/bin/gcc /usr/bin/gcc \
-    && rm /usr/bin/g++ \
-    && ln -s /usr/local/bin/g++ /usr/bin/g++ \
-    && rm /usr/lib/x86_64-linux-gnu/libstdc++.so.6 \
-    && ln -s /usr/local/lib64/libstdc++.so.6.0.30 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
+#RUN wget http://ftp.gnu.org/gnu/gcc/gcc-12.2.0/gcc-12.2.0.tar.gz \
+#    && tar -zxvf gcc-12.2.0.tar.gz \
+#    && cd gcc-12.2.0 \
+#    && ./contrib/download_prerequisites \
+#    && mkdir build \
+#    && cd build/ \
+#    && ../configure -enable-checking=release -enable-languages=c,c++ -disable-multilib \
+#    && make -j4 \
+#    && make install \
+#    && rm /usr/bin/gcc \
+#    && ln -s /usr/local/bin/gcc /usr/bin/gcc \
+#    && rm /usr/bin/g++ \
+#    && ln -s /usr/local/bin/g++ /usr/bin/g++ \
+#    && rm /usr/lib/x86_64-linux-gnu/libstdc++.so.6 \
+#    && ln -s /usr/local/lib64/libstdc++.so.6.0.30 /usr/lib/x86_64-linux-gnu/libstdc++.so.6
 
 RUN mkdir /u01/ \
     \
@@ -135,12 +135,12 @@ RUN cd /tmp/vectordb && \
 	make install
 
 # the followings two commands install an http-client library called curlpp
-RUN apt-get update && apt-get install -y libcurl4-openssl-dev pkg-config
-RUN git clone https://github.com/jpbarrette/curlpp.git && cd curlpp && \
-    git reset --hard  592552a && \
-    mkdir build && cd build && \
-    cmake .. && make -j$(nproc) && \
-    make install
+#RUN apt-get update && apt-get install -y libcurl4-openssl-dev pkg-config
+#RUN git clone https://github.com/jpbarrette/curlpp.git && cd curlpp && \
+#    git reset --hard  592552a && \
+#    mkdir build && cd build && \
+#    cmake .. && make -j$(nproc) && \
+#    make install
 
 ENV PATH="${PATH}:/usr/local/lib"
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
@@ -149,4 +149,5 @@ ENV LANG en_US.utf8
 USER postgres
 EXPOSE 5432
 ENTRYPOINT ["docker-entrypoint.sh"]
+#ENTRYPOINT ["sleep","infinity"]
 

README.md

@@ -1,11 +1,15 @@
 # MSVBASE
+## What's NEW!
+New Research Paper [VBASE: Unifying Online Vector Similarity Search and Relational Queries via Relaxed Monotonicity](https://www.usenix.org/system/files/osdi23-zhang-qianxi_1.pdf) - _published in OSDI 2023_
+## What's MSVBASE!
+MSVBASE is a new system capable of efficiently serving complex online queries that involve both approximate similarity search and relational operators on scalar and vector data-sets. MSVBASE builds a unified query execution engine to support a wide range of queries both on scalar and vector data, and shows superior query performance and accuracy.
+It integrates high-dimensional vector indices into PostgreSQL with minimal code modifications.
 
-MSVBASE is a system that efficiently supports complex queries of both approximate similarity search and relational operators. It integrates high-dimensional vector indices into PostgreSQL, a relational database to facilitate complex approximate similarity queries.
-
-## **Build Docker**
+## **Quickstart**
 ### **Clone and Patch**
 ```
 git clone https://github.com/microsoft/MSVBASE.git
+cd MSVBASE
 git submodule update --init --recursive
 ./scripts/patch.sh
 ```
@@ -16,28 +20,75 @@ git submodule update --init --recursive
 
 ### **Run**
 ```
-./scripts/dockerbuild.sh
+./scripts/dockerrun.sh
 ```
 
-## **SQL**
-It is compatible with PostgreSQL syntax and protocols, supporting vector distance calculations for L2 and Inner product. It also supports hnsw and sptag indexes. Soon, we will be introducing spann and more indexes. Stay tuned!
+## **Features and SQL syntax**
+It is compatible with PostgreSQL syntax and protocol, supporting vector distance calculations for L2 and Inner product. It also supports [hnsw](https://github.com/nmslib/hnswlib) and [sptag](https://github.com/microsoft/SPTAG/) indices. Soon, we are going to integrate spann and more indices.
 ### **Command Line**
 ```
 docker exec -it --privileged --user=root vbase_open_source bash
 psql -U vectordb
 ```
-### **Example**
+
+### **SQL syntax**
+It preserves all the features of PostgreSQL while extending query support on vector data.
+* Currently, 'float array' is used to store vectors.
+```
+create table t_table(id int, price int, vector_1 float8[10], vector_2 float8[10]);
+```
+* Insert or Import data.
+```
+insert into t_table values(1, 10, '{1,2,3,4,5,6,7,8,9,0}', '{5,6,7,1,2,3,4,8,9,1}');
+copy t_table from 'your_data_path.tsv' DELIMITER E'\t' csv quote e'\x01';
+```
+* When creating a vector index, it is necessary to specify the algorithm and the distance calculation method to be used.
+```
+create index vector_index_1 on t_table using hnsw(vector_1) with(dimension=10,distmethod=l2_distance);
+create index vector_index_2 on t_table using sptag(vector_2) with(dimension=10,distmethod=inner_product_distance);
+```
+* **TopK**. When calculating distances, the '<->' operator represents the L2 distance, while '<*>' represents the inner product distance.
+```
+select id from t_table order by vector_1 <-> '{5,9,8,6,2,1,1,0,4,3}' limit 10;
+select id from t_table order by vector_2 <*> '{5,9,8,6,2,1,1,0,4,3}' limit 5;
+```
+* **TopK + Filter**.
+```
+select id from t_table where price > 15 order by vector_1 <-> '{5,9,8,6,2,1,1,0,4,3}' limit 10;
+select id from t_table where price > 15 order by vector_2 <*> '{5,9,8,6,2,1,1,0,4,3}' limit 5;
+```
+* **Distance Range Filter**. It also supports distance threshold-based filtering queries. The query will retrieve vector data that is within the distance threshold.
+In the query, the '<<->>' operator represents the L2 distance, while '<<*>>' represents the inner product distance.
+The first element of the array represents the distance threshold.
+```
+select id from t_table where price > 15 and vector_1 <<->> '{30,5,9,8,6,2,1,1,0,4,3}';
+```
+* **Multi-vector Column Query**.
+```
+select id from t_table
+order by approximate_sum('0.5 * vector_1<->{5,9,8,6,2,1,1,0,4,3} + vector_2<*>{5,9,8,6,2,1,1,0,4,3}' ) limit 5;
+```
+* **Join**. Join on vector similarity with threshold
+```
+select t_table.id as tid, d_table.id as did
+from t_table join d_table
+on t_table.vector_2 <<*>> array_cat(ARRAY[cast(10 as float8)], d_table.vector_2);
+```
+
+* Example
 ```
 create database test;
 \c test;
 create extension vectordb;
-create table t_table(id int, price int, m_vector_1 float8[10]);
+create table t_table(id int, price int, vector_1 float8[10]);
 insert into t_table values(1, 10, '{1,2,3,4,5,6,7,8,9,0}');
 insert into t_table values(2, 20, '{5,6,7,1,2,3,4,8,9,1}');
 insert into t_table values(3, 30, '{9,8,7,6,5,4,3,2,1,0}');
-create index t4_index on t_table using hnsw(m_vector_1) with(dimension=10,distmethod=l2_distance);
+create index t4_index on t_table using hnsw(vector_1) with(dimension=10,distmethod=l2_distance);
 set enable_seqscan=false;
-select id from t_table where price > 15 order by m_vector_1 <-> '{5,9,8,6,2,1,1,0,4,3}' limit 1;
+select id from t_table where price > 15 order by vector_1 <-> '{5,9,8,6,2,1,1,0,4,3}' limit 1;
+insert into t_table values(4, 40, '{19,18,17,16,15,14,13,12,11,10}');
+delete from t_table where id = 2;
 ```
 
 ## Contributing