diff --git a/.gitignore b/.gitignore index c30610f..67313d4 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,6 @@ ThirdParty/stxxl .idea/* build/* + +Cargo.lock +target diff --git a/.travis.yml b/.travis.yml index c611de8..ab68df0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,22 +5,36 @@ compiler: - g++ os: - linux - - -before_install: - - sudo apt-get -qq update - - sudo apt-get install -y make automake autotools-dev cmake -script: - - mkdir -p build - - cd build - - cmake .. - - make - - ctest - - gcov -n -o . src/gqf.cpp > /dev/null; branches: only: - mqfDevelopmenet - master +cache: + directories: + - "$HOME/.cargo" + - "target" -after_success: - - bash <(curl -s https://codecov.io/bash) +jobs: + include: + - &test + stage: test + before_install: + - sudo apt-get -qq update + - sudo apt-get install -y make automake autotools-dev cmake + script: + - mkdir -p build + - cd build + - cmake .. + - make + - ctest + - gcov -n -o . src/gqf.cpp > /dev/null; + after_success: + - bash <(curl -s https://codecov.io/bash) + - <<: *test + name: Rust FFI bindings + language: rust + rust: stable + env: + - RUSTFLAGS="-Clink-arg=-fuse-ld=gold" + script: + - cargo test diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..35d667a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "mqf" +version = "0.1.0" +authors = ["Luiz Irber "] +links = "libmqf" + +[build-dependencies] +bindgen = "0.51" +cmake = "0.1.42" + +[dev-dependencies] +tempfile = "3.1.0" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..bc895ac --- /dev/null +++ b/build.rs @@ -0,0 +1,65 @@ +use std::env; +use std::path::PathBuf; + +extern crate cmake; +use cmake::Config; + +fn main() { + let dst = Config::new(".") + .define("BUILD_STATIC_LIBS", "ON") + .build_target("MQF") + .build(); + + // TODO: there are probably better ways to do this... + let target = env::var("TARGET").unwrap(); + if target.contains("apple") { + println!("cargo:rustc-link-lib=dylib=c++"); + } else if target.contains("linux") { + println!("cargo:rustc-link-lib=dylib=stdc++"); + } else { + unimplemented!(); + } + + println!("cargo:rustc-link-search=native={}/build/src", dst.display()); + println!("cargo:rustc-link-lib=static=MQF"); + + println!( + "cargo:rustc-link-search=native={}/build/ThirdParty/stxxl/lib", + dst.display() + ); + + let mode = match env::var("PROFILE").unwrap().as_ref() { + "debug" => "_debug", + _ => "", + }; + println!("cargo:rustc-link-lib=static=stxxl{}", mode); + + let bindings = bindgen::Builder::default() + .clang_arg("-I./include") + .clang_arg("-x") + .clang_arg("c++") + .clang_arg("-std=c++11") + .header("include/gqf.h") + .whitelist_type("QF") + .whitelist_type("QFi") + .whitelist_function("qf_init") + .whitelist_function("qf_insert") + .whitelist_function("qf_count_key") + .whitelist_function("qf_destroy") + .whitelist_function("qf_copy") + .whitelist_function("qf_serialize") + .whitelist_function("qf_deserialize") + .whitelist_function("qf_migrate") + .whitelist_function("qf_iterator") + .whitelist_function("qfi_get") + .whitelist_function("qfi_next") + .whitelist_function("qfi_end") + .blacklist_type("std::*") + .generate() + .expect("Unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write bindings!"); +} diff --git a/include/gqf.h b/include/gqf.h index abe5f5b..9869f9f 100644 --- a/include/gqf.h +++ b/include/gqf.h @@ -109,7 +109,7 @@ extern "C" { void qf_destroy(QF *qf); - void qf_copy(QF *dest, QF *src); + void qf_copy(QF *dest, const QF *src); /*! @breif Increment the counter for this item by count. diff --git a/src/gqf.cpp b/src/gqf.cpp index 60b583c..f17769b 100644 --- a/src/gqf.cpp +++ b/src/gqf.cpp @@ -1982,7 +1982,7 @@ bool qf_getBlockLabel_pointer_byItem(const QF *qf, uint64_t key,char *&res){ /* The caller should call qf_init on the dest QF before calling this function. */ -void qf_copy(QF *dest, QF *src) +void qf_copy(QF *dest, const QF *src) { memcpy(dest->mem, src->mem, sizeof(qfmem)); memcpy(dest->metadata, src->metadata, sizeof(qfmetadata)); diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..f6f1599 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,223 @@ +mod raw; + +use std::ffi::CString; +use std::path::Path; +use std::ptr; + +#[derive(Debug)] +pub struct MQF { + inner: raw::QF, +} + +impl Default for MQF { + fn default() -> MQF { + MQF { + inner: raw::QF { + mem: ptr::null_mut(), + metadata: ptr::null_mut(), + blocks: ptr::null_mut(), + }, + } + } +} + +impl Drop for MQF { + fn drop(&mut self) { + unsafe { raw::qf_destroy(&mut self.inner) }; + } +} + +impl Clone for MQF { + fn clone(&self) -> Self { + let mut new_qf = MQF::default(); + unsafe { + raw::qf_copy(&mut new_qf.inner, &self.inner); + }; + new_qf + } +} + +unsafe impl Sync for MQF {} + +impl MQF { + pub fn new(counter_size: u64, qbits: u64) -> MQF { + let mut mqf = MQF::default(); + + let num_hash_bits = qbits + 8; + + let s = CString::new("").unwrap(); + + unsafe { + raw::qf_init( + &mut mqf.inner, + 1u64 << qbits, // nslots + num_hash_bits, // key_bits + 0, // label_bits + counter_size, // fixed_counter_size + 0, // blocksLabelSize + true, // mem + s.as_ptr(), // path + 2038074760, // seed (doesn't matter) + ); + }; + + mqf + } + + pub fn insert(&mut self, key: u64, count: u64) { + unsafe { raw::qf_insert(&mut self.inner, key, count, true, true) }; + //unsafe { raw::qf_insert(&mut self.inner, key, count, false, false) }; + } + + pub fn count_key(&self, key: u64) -> u64 { + unsafe { raw::qf_count_key(&self.inner, key) } + } + + pub fn iter(&mut self) -> MQFIter { + let mut cfi = raw::QFi { + qf: ptr::null_mut(), + run: 0, + current: 0, + cur_start_index: 0, + cur_length: 0, + num_clusters: 0, + c_info: ptr::null_mut(), + }; + + // TODO: treat false + let _ = unsafe { raw::qf_iterator(&mut self.inner, &mut cfi, 0) }; + + MQFIter { inner: cfi } + } + + pub fn serialize>(&self, path: P) -> Result<(), Box> { + let s = CString::new(path.as_ref().to_str().unwrap())?; + + unsafe { + raw::qf_serialize(&self.inner, s.as_ptr()); + } + + Ok(()) + } + + pub fn deserialize>(path: P) -> Result> { + let mut qf = MQF::default(); + let s = CString::new(path.as_ref().to_str().unwrap())?; + + unsafe { + raw::qf_deserialize(&mut qf.inner, s.as_ptr()); + } + + Ok(qf) + } + + pub fn merge(&mut self, other: &mut MQF) -> Result<(), Box> { + unsafe { + raw::qf_migrate(&mut other.inner, &mut self.inner); + } + + Ok(()) + } +} + +pub struct MQFIter { + inner: raw::QFi, +} + +impl Iterator for MQFIter { + type Item = (u64, u64, u64); + + fn next(&mut self) -> Option { + if unsafe { raw::qfi_end(&mut self.inner) } != 0 { + None + } else { + let mut key = 0; + let mut value = 0; + let mut count = 0; + + unsafe { + raw::qfi_get(&mut self.inner, &mut key, &mut value, &mut count); + raw::qfi_next(&mut self.inner) + }; + Some((key, value, count)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn simple_counting_test_api() { + //except first item is inserted 5 times to full test _insert1 + let counter_size = 2; + let qbits = 5; + let mut qf: MQF = MQF::new(counter_size, qbits); + + let mut count = 0; + let mut fixed_counter = 0; + + for i in 0..=10 { + qf.insert(100, 1); + count = qf.count_key(100); + dbg!((count, fixed_counter)); + assert_eq!(count, 1 + i); + } + + qf.insert(1500, 50); + + count = qf.count_key(1500); + dbg!((count, fixed_counter)); + assert_eq!(count, 50); + + qf.insert(1600, 60); + count = qf.count_key(1600); + dbg!((count, fixed_counter)); + assert_eq!(count, 60); + + qf.insert(2000, 4000); + count = qf.count_key(2000); + dbg!((count, fixed_counter)); + assert_eq!(count, 4000); + } + + #[test] + fn big_count() { + let mut qf = MQF::new(4, 5); + qf.insert(100, 100000); + let mut count = qf.count_key(100); + assert_eq!(count, 100000); + } + + #[test] + fn iter_next() { + let mut qf = MQF::new(4, 5); + qf.insert(100, 100000); + qf.insert(101, 10000); + qf.insert(102, 1000); + qf.insert(103, 100); + + let vals: Vec<(u64, u64, u64)> = qf.iter().collect(); + dbg!(&vals); + assert_eq!(vals.len(), 4); + } + + #[test] + fn serde() { + let mut qf = MQF::new(4, 5); + qf.insert(100, 100000); + qf.insert(101, 10000); + qf.insert(102, 1000); + qf.insert(103, 100); + + let vals: Vec<(u64, u64, u64)> = qf.iter().collect(); + + let mut file = tempfile::NamedTempFile::new().unwrap(); + qf.serialize(file.path()).unwrap(); + + let mut new_qf = MQF::deserialize(file.path()).unwrap(); + let new_vals: Vec<(u64, u64, u64)> = new_qf.iter().collect(); + assert_eq!(vals, new_vals); + } +} diff --git a/src/raw.rs b/src/raw.rs new file mode 100644 index 0000000..b199b4a --- /dev/null +++ b/src/raw.rs @@ -0,0 +1,81 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(dead_code)] +#![allow(improper_ctypes)] +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +// TODO: eventually want to remove the `improper_ctypes` lint, +// but MQF is using uint128_t internally... + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::ptr; + + #[test] + fn simple_counting_test() { + //except first item is inserted 5 times to full test _insert1 + let mut qf: QF = QF { + mem: ptr::null_mut(), + metadata: ptr::null_mut(), + blocks: ptr::null_mut(), + }; + + let counter_size = 2; + let qbits = 5; + let num_hash_bits = qbits + 8; + let maximum_count = (1u64 << counter_size) - 1; + let mut count = 0; + let mut fixed_counter = 0; + + let s = CString::new("").unwrap(); + + //INFO("Counter size = "<