ARROW-259: Use Flatbuffer Field type instead of MaterializedField

Remove MaterializedField, MajorType, RepeatedTypes Add code to convert from FlatBuf representation to Pojo also adds tests to test the conversion
apache · Aug 18, 2016 · e7e399d · wesm · Aug 19, 2016 · julienledem
1 parent 246a126
commit e7e399d
Show file tree

Hide file tree

Showing 77 changed files with 2,464 additions and 5,180 deletions.
diff --git a/format/Message.fbs b/format/Message.fbs
@@ -1,10 +1,13 @@
-namespace apache.arrow.flatbuf;
+namespace org.apache.arrow.flatbuf;
 
 /// ----------------------------------------------------------------------
 /// Logical types and their metadata (if any)
 ///
 /// These are stored in the flatbuffer in the Type union below
 
+table Null {
+}
+
 /// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
 /// (according to the physical memory layout). We used Tuple here as Struct is
 /// a reserved word in Flatbuffers
@@ -45,10 +48,22 @@ table Decimal {
   scale: int;
 }
 
+table Date {
+}
+
+table Time {
+}
+
 table Timestamp {
   timezone: string;
 }
 
+table IntervalDay {
+}
+
+table IntervalYear {
+}
+
 table JSONScalar {
   dense:bool=true;
 }
@@ -58,13 +73,18 @@ table JSONScalar {
 /// add new logical types to Type without breaking backwards compatibility
 
 union Type {
+  Null,
   Int,
   FloatingPoint,
   Binary,
   Utf8,
   Bool,
   Decimal,
+  Date,
+  Time,
   Timestamp,
+  IntervalDay,
+  IntervalYear,
   List,
   Tuple,
   Union,

diff --git a/header b/header
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
diff --git a/java/format/pom.xml b/java/format/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+  license agreements. See the NOTICE file distributed with this work for additional 
+  information regarding copyright ownership. The ASF licenses this file to 
+  You under the Apache License, Version 2.0 (the "License"); you may not use 
+  this file except in compliance with the License. You may obtain a copy of 
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+  by applicable law or agreed to in writing, software distributed under the 
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+  OF ANY KIND, either express or implied. See the License for the specific 
+  language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+<parent>
+  <artifactId>arrow-java-root</artifactId>
+  <groupId>org.apache.arrow</groupId>
+  <version>0.1-decimal</version>
+</parent>
+
+<artifactId>arrow-format</artifactId>
+<packaging>jar</packaging>
+<name>Arrow Format</name>
+
+  <properties>
+    <fbs.version>1.2.0-3f79e055</fbs.version>
+    <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
+    <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
+    <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.vlkan</groupId>
+      <artifactId>flatbuffers</artifactId>
+      <version>${fbs.version}</version>
+    </dependency>
+  </dependencies>
+
+ <build>
+  <extensions>
+    <!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
+    <extension>
+      <groupId>kr.motd.maven</groupId>
+      <artifactId>os-maven-plugin</artifactId>
+      <version>${os-maven-plugin.version}</version>
+    </extension>
+  </extensions>
+
+  <plugins>
+    <plugin>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-dependency-plugin</artifactId>
+      <version>${maven-dependency-plugin.version}</version>
+      <executions>
+        <execution>
+          <id>copy-flatc</id>
+          <phase>generate-sources</phase>
+          <goals>
+            <goal>copy</goal>
+          </goals>
+          <configuration>
+            <artifactItems>
+              <artifactItem>
+                <groupId>com.vlkan</groupId>
+                <artifactId>flatc-${os.detected.classifier}</artifactId>
+                <version>${fbs.version}</version>
+                <type>exe</type>
+                <overWrite>true</overWrite>
+                <outputDirectory>${project.build.directory}</outputDirectory>
+              </artifactItem>
+            </artifactItems>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.codehaus.mojo</groupId>
+      <artifactId>exec-maven-plugin</artifactId>
+      <version>1.4.0</version>
+      <executions>
+        <execution>
+          <id>script-chmod</id>
+          <goals>
+            <goal>exec</goal>
+          </goals>
+          <phase>generate-sources</phase>
+          <configuration>
+            <executable>chmod</executable>
+            <arguments>
+              <argument>+x</argument>
+              <argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
+            </arguments>
+          </configuration>
+        </execution>
+        <execution>
+          <goals>
+            <goal>exec</goal>
+          </goals>
+          <phase>generate-sources</phase>
+          <configuration>
+            <executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
+            <arguments>
+              <argument>-j</argument>
+              <argument>-o</argument>
+              <argument>target/generated-sources/</argument>
+              <argument>../../format/Message.fbs</argument>
+            </arguments>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>com.mycila</groupId>
+      <artifactId>license-maven-plugin</artifactId>
+      <version>2.3</version>
+      <configuration>
+        <header>${basedir}/../../header</header>
+        <includes>
+          <include>**/*.java</include>
+        </includes>
+      </configuration>
+      <executions>
+        <execution>
+          <phase>process-sources</phase>
+          <goals>
+            <goal>format</goal>
+          </goals>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.codehaus.mojo</groupId>
+      <artifactId>build-helper-maven-plugin</artifactId>
+      <version>1.9.1</version>
+      <executions>
+        <execution>
+          <id>add-sources-as-resources</id>
+          <phase>generate-sources</phase>
+          <goals>
+            <goal>add-source</goal>
+          </goals>
+          <configuration>
+            <sources>
+              <source>${project.build.directory}/generated-sources</source>
+            </sources>
+          </configuration>
+        </execution>
+      </executions>
+    </plugin>
+    <plugin>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-checkstyle-plugin</artifactId>
+      <configuration>
+        <skip>true</skip>
+      </configuration>
+    </plugin>
+  </plugins>
+
+</build>
+</project>
+
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
@@ -15,7 +15,7 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1-decimal</version>
   </parent>
   <artifactId>arrow-memory</artifactId>
   <name>arrow-memory</name>

diff --git a/java/pom.xml b/java/pom.xml
@@ -21,7 +21,7 @@
 
   <groupId>org.apache.arrow</groupId>
   <artifactId>arrow-java-root</artifactId>
-  <version>0.1-SNAPSHOT</version>
+  <version>0.1-decimal</version>
   <packaging>pom</packaging>
 
   <name>Apache Arrow Java Root POM</name>
@@ -465,6 +465,7 @@
   </dependencies>
 
   <modules>
+    <module>format</module>
     <module>memory</module>
     <module>vector</module>
   </modules>

diff --git a/java/vector/pom.xml b/java/vector/pom.xml
@@ -15,13 +15,18 @@
   <parent>
     <groupId>org.apache.arrow</groupId>
     <artifactId>arrow-java-root</artifactId>
-    <version>0.1-SNAPSHOT</version>
+    <version>0.1-decimal</version>
   </parent>
   <artifactId>vector</artifactId>
   <name>vectors</name>
 
   <dependencies>
 
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-format</artifactId>
+      <version>0.1-decimal</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory</artifactId>

diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
@@ -17,6 +17,7 @@
 data: {
     # TODO:  Rename to ~valueVectorModesAndTypes for clarity.
     vv:                       tdd(../data/ValueVectorTypes.tdd),
+    arrowTypes:               tdd(../data/ArrowTypes.tdd)
 
 }
 freemarkerLinks: {

diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+  types: [
+    {
+      name: "Null",
+      fields: []
+    },
+    {
+      name: "Tuple",
+      fields: []
+    },
+    {
+      name: "List",
+      fields: []
+    },
+    {
+      name: "Union",
+      fields: []
+    },
+    {
+      name: "Int",
+      fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
+    },
+    {
+      name: "FloatingPoint",
+      fields: [{name: precision, type: int}]
+    },
+    {
+      name: "Utf8",
+      fields: []
+    },
+    {
+      name: "Binary",
+      fields: []
+    },
+    {
+      name: "Bool",
+      fields: []
+    },
+    {
+      name: "Decimal",
+      fields: [{name: "precision", type: int}, {name: "scale", type: int}]
+    },
+    {
+      name: "Date",
+      fields: []
+    },
+    {
+      name: "Time",
+      fields: []
+    },
+    {
+      name: "Timestamp",
+      fields: [{name: "timezone", type: "String"}]
+    },
+    {
+      name: "IntervalDay",
+      fields: []
+    },
+    {
+      name: "IntervalYear",
+      fields: []
+    }
+  ]
+}