Merge pull request #11 from alecmocatta/clean

Clean up

.editorconfig

@@ -0,0 +1,5 @@
+[*]
+tab_width = 4
+
+[.*]
+tab_width = 4

.gitmodules

@@ -2,3 +2,11 @@
 	path = amadeus-testing
 	url = https://github.com/alecmocatta/amadeus-testing.git
 	branch = master
+[submodule "amadeus-parquet/thrift"]
+	path = amadeus-parquet/thrift
+	url = https://github.com/apache/thrift.git
+	branch = master
+[submodule "amadeus-parquet/parquet-format"]
+	path = amadeus-parquet/parquet-format
+	url = https://github.com/apache/parquet-format.git
+	branch = master

.mergify.yml

@@ -7,7 +7,6 @@ pull_request_rules:
       - "#approved-reviews-by>=1"
       - "#review-requested=0"
       - "#changes-requested-reviews-by=0"
-      - "#commented-reviews-by=0"
     actions:
       merge:
         method: merge
@@ -21,7 +20,6 @@ pull_request_rules:
       - author=alecmocatta # https://github.com/Mergifyio/mergify-engine/issues/451
       - "#review-requested=0"
       - "#changes-requested-reviews-by=0"
-      - "#commented-reviews-by=0"
     actions:
       merge:
         method: merge

Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "amadeus"
-version = "0.1.3"
+version = "0.1.4"
 license = "Apache-2.0"
 authors = ["Alec Mocatta <[email protected]>"]
 categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
@@ -14,7 +14,7 @@ parquet postgres aws s3 cloudfront elb json csv logs hadoop hdfs arrow common cr
 """
 repository = "https://github.com/alecmocatta/amadeus"
 homepage = "https://github.com/alecmocatta/amadeus"
-documentation = "https://docs.rs/amadeus/0.1.3"
+documentation = "https://docs.rs/amadeus/0.1.4"
 readme = "README.md"
 edition = "2018"
 
@@ -36,23 +36,20 @@ doc = ["amadeus-aws/doc", "amadeus-commoncrawl/doc", "amadeus-parquet/doc", "ama
 features = ["doc", "constellation", "aws", "commoncrawl", "postgres", "csv", "json"]
 
 [dependencies]
-amadeus-core = { version = "=0.1.3", path = "amadeus-core" }
-amadeus-derive = { version = "=0.1.3", path = "amadeus-derive" }
-amadeus-types = { version = "=0.1.3", path = "amadeus-types" }
-amadeus-aws = { version = "=0.1.3", path = "amadeus-aws", optional = true }
-amadeus-commoncrawl = { version = "=0.1.3", path = "amadeus-commoncrawl", optional = true }
-amadeus-parquet = { version = "=0.1.3", path = "amadeus-parquet", optional = true }
-amadeus-postgres = { version = "=0.1.3", path = "amadeus-postgres", optional = true }
-amadeus-serde = { version = "=0.1.3", path = "amadeus-serde", optional = true }
+amadeus-core = { version = "=0.1.4", path = "amadeus-core" }
+amadeus-derive = { version = "=0.1.4", path = "amadeus-derive" }
+amadeus-types = { version = "=0.1.4", path = "amadeus-types" }
+amadeus-aws = { version = "=0.1.4", path = "amadeus-aws", optional = true }
+amadeus-commoncrawl = { version = "=0.1.4", path = "amadeus-commoncrawl", optional = true }
+amadeus-parquet = { version = "=0.1.4", path = "amadeus-parquet", optional = true }
+amadeus-postgres = { version = "=0.1.4", path = "amadeus-postgres", optional = true }
+amadeus-serde = { version = "=0.1.4", path = "amadeus-serde", optional = true }
 constellation-rs = { version = "0.1", default-features = false, optional = true }
 futures-preview = "=0.3.0-alpha.19"
 pin-utils = "0.1.0-alpha.4"
 serde = { version = "1.0", features = ["derive"] }
 serde_closure = "0.2"
-serde_traitobject = "0.1.6"
-
-# pin; broken for some reason
-bincode = "=1.0.1"
+serde_traitobject = "0.2"
 
 [dev-dependencies]
 either = { version = "1.5", features = ["serde"] }

README.md

@@ -13,7 +13,7 @@
 </p>
 
 <p align="center">
-	<a href="https://docs.rs/amadeus/0.1.3">Docs</a>
+	<a href="https://docs.rs/amadeus/0.1.4">Docs</a>
 </p>
 
 

amadeus-aws/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "amadeus-aws"
-version = "0.1.3"
+version = "0.1.4"
 license = "Apache-2.0"
 authors = ["Alec Mocatta <[email protected]>"]
 categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
@@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
 """
 repository = "https://github.com/alecmocatta/amadeus"
 homepage = "https://github.com/alecmocatta/amadeus"
-documentation = "https://docs.rs/amadeus/0.1.3"
+documentation = "https://docs.rs/amadeus/0.1.4"
 readme = "README.md"
 edition = "2018"
 
@@ -22,8 +22,8 @@ maintenance = { status = "actively-developed" }
 doc = []
 
 [dependencies]
-amadeus-core = { version = "=0.1.3", path = "../amadeus-core" }
-amadeus-types = { version = "=0.1.3", path = "../amadeus-types" }
+amadeus-core = { version = "=0.1.4", path = "../amadeus-core" }
+amadeus-types = { version = "=0.1.4", path = "../amadeus-types" }
 chrono = { version = "0.4", default-features = false }
 flate2 = "1.0"
 futures-01 = { package = "futures", version = "0.1" }

amadeus-aws/src/cloudfront.rs

@@ -59,6 +59,7 @@ impl Source for Cloudfront {
 			.into_dist_iter()
 			.flat_map(FnMut!(move |key: String| {
 				let client = S3Client::new(region.clone());
+				let mut errors = 0;
 				ResultExpand(
 					loop {
 						match self::block_on_01(self::retry(|| {
@@ -68,11 +69,15 @@ impl Source for Cloudfront {
 								..GetObjectRequest::default()
 							})
 						})) {
-							Err(RusotoError::HttpDispatch(_)) => continue,
+							Err(RusotoError::HttpDispatch(_)) if errors < 10 => {
+								errors += 1;
+								continue;
+							}
 							Err(RusotoError::Unknown(response))
-								if response.status.is_server_error() =>
+								if response.status.is_server_error() && errors < 10 =>
 							{
-								continue
+								errors += 1;
+								continue;
 							}
 							res => break res,
 						}

amadeus-aws/src/file.rs

@@ -206,6 +206,7 @@ impl Page for S3Page {
 		Box::pin(async move {
 			let len: u64 = buf.len().try_into().unwrap();
 			let mut cursor = io::Cursor::new(buf);
+			let mut errors = 0;
 			while len - cursor.position() > 0 {
 				let (start, end) = (offset + cursor.position(), offset + len - 1);
 				let res = Compat01As03::new(self.client.get_object(GetObjectRequest {
@@ -216,9 +217,15 @@ impl Page for S3Page {
 				}));
 				let res = res.await;
 				match res {
-					Err(RusotoError::HttpDispatch(_)) => continue,
-					Err(RusotoError::Unknown(response)) if response.status.is_server_error() => {
-						continue
+					Err(RusotoError::HttpDispatch(_)) if errors < 10 => {
+						errors += 1;
+						continue;
+					}
+					Err(RusotoError::Unknown(response))
+						if response.status.is_server_error() && errors < 10 =>
+					{
+						errors += 1;
+						continue;
 					}
 					_ => (),
 				}

amadeus-aws/src/lib.rs

@@ -1,4 +1,4 @@
-#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.1.3")]
+#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.1.4")]
 #![feature(type_alias_impl_trait)]
 
 mod cloudfront;

amadeus-commoncrawl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "amadeus-commoncrawl"
-version = "0.1.3"
+version = "0.1.4"
 license = "MIT OR Apache-2.0"
 authors = ["Stephen Becker IV <[email protected]>", "Alec Mocatta <[email protected]>"]
 categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
@@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
 """
 repository = "https://github.com/alecmocatta/amadeus"
 homepage = "https://github.com/alecmocatta/amadeus"
-documentation = "https://docs.rs/amadeus/0.1.3"
+documentation = "https://docs.rs/amadeus/0.1.4"
 readme = "README.md"
 edition = "2018"
 
@@ -22,8 +22,8 @@ maintenance = { status = "actively-developed" }
 doc = []
 
 [dependencies]
-amadeus-core = { version = "=0.1.3", path = "../amadeus-core" }
-amadeus-types = { version = "=0.1.3", path = "../amadeus-types" }
+amadeus-core = { version = "=0.1.4", path = "../amadeus-core" }
+amadeus-types = { version = "=0.1.4", path = "../amadeus-types" }
 flate2 = "1.0"
 reqwest = "0.9"
 reqwest_resume = "0.2"

amadeus-commoncrawl/src/lib.rs

@@ -1,4 +1,4 @@
-#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.1.3")]
+#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.1.4")]
 #![feature(type_alias_impl_trait)]
 
 mod commoncrawl;

amadeus-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "amadeus-core"
-version = "0.1.3"
+version = "0.1.4"
 license = "Apache-2.0"
 authors = ["Alec Mocatta <[email protected]>"]
 categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
@@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
 """
 repository = "https://github.com/alecmocatta/amadeus"
 homepage = "https://github.com/alecmocatta/amadeus"
-documentation = "https://docs.rs/amadeus/0.1.3"
+documentation = "https://docs.rs/amadeus/0.1.4"
 readme = "README.md"
 edition = "2018"