Skip to content

Commit ad9a472

Browse files
dfa1claude
dfa1
and
claude
committed
feat(csv): add csv module with CsvImporter and CsvExporter
Uses FastCSV 3.6.0 (JPMS-compatible). Importer infers column types (long → double → bool → utf8) with optional schema override. Exporter supports Path and Writer destinations. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 42c4b55 commit ad9a472

9 files changed

Lines changed: 605 additions & 0 deletions

File tree

‎csv/pom.xml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xmlns="http://maven.apache.org/POM/4.0.0"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<parent>
7+
<groupId>io.github.dfa1.vortex</groupId>
8+
<artifactId>vortex-java</artifactId>
9+
<version>0.1.0-SNAPSHOT</version>
10+
</parent>
11+
12+
<artifactId>csv</artifactId>
13+
14+
<dependencies>
15+
<dependency>
16+
<groupId>io.github.dfa1.vortex</groupId>
17+
<artifactId>reader</artifactId>
18+
</dependency>
19+
<dependency>
20+
<groupId>io.github.dfa1.vortex</groupId>
21+
<artifactId>writer</artifactId>
22+
</dependency>
23+
<dependency>
24+
<groupId>de.siegmar</groupId>
25+
<artifactId>fastcsv</artifactId>
26+
</dependency>
27+
<dependency>
28+
<groupId>org.junit.jupiter</groupId>
29+
<artifactId>junit-jupiter</artifactId>
30+
</dependency>
31+
<dependency>
32+
<groupId>org.assertj</groupId>
33+
<artifactId>assertj-core</artifactId>
34+
</dependency>
35+
<dependency>
36+
<groupId>org.mockito</groupId>
37+
<artifactId>mockito-junit-jupiter</artifactId>
38+
</dependency>
39+
</dependencies>
40+
</project>

‎csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package io.github.dfa1.vortex.csv;
2+
3+
import de.siegmar.fastcsv.writer.CsvWriter;
4+
import io.github.dfa1.vortex.core.DType;
5+
import io.github.dfa1.vortex.core.VortexException;
6+
import io.github.dfa1.vortex.core.array.BoolArray;
7+
import io.github.dfa1.vortex.core.array.ByteArray;
8+
import io.github.dfa1.vortex.core.array.DoubleArray;
9+
import io.github.dfa1.vortex.core.array.FloatArray;
10+
import io.github.dfa1.vortex.core.array.IntArray;
11+
import io.github.dfa1.vortex.core.array.LongArray;
12+
import io.github.dfa1.vortex.core.array.ShortArray;
13+
import io.github.dfa1.vortex.core.array.VarBinArray;
14+
import io.github.dfa1.vortex.core.array.Array;
15+
import io.github.dfa1.vortex.io.VortexReader;
16+
import io.github.dfa1.vortex.scan.ScanIterator;
17+
import io.github.dfa1.vortex.scan.ScanOptions;
18+
import io.github.dfa1.vortex.scan.ScanResult;
19+
20+
import java.io.FilterWriter;
21+
import java.io.IOException;
22+
import java.io.Writer;
23+
import java.nio.charset.StandardCharsets;
24+
import java.nio.file.Path;
25+
import java.util.List;
26+
27+
/// Reads a Vortex file and writes rows to a CSV destination.
28+
///
29+
/// The header row is derived from [DType.Struct] field names.
30+
/// Only struct root dtype is supported; throws [VortexException] otherwise.
31+
public final class CsvExporter {
32+
33+
private CsvExporter() {
34+
}
35+
36+
public static void exportCsv(Path vortexPath, Path csvPath) throws IOException {
37+
exportCsv(vortexPath, csvPath, ExportOptions.defaults());
38+
}
39+
40+
public static void exportCsv(Path vortexPath, Path csvPath, ExportOptions options) throws IOException {
41+
try (VortexReader reader = VortexReader.open(vortexPath);
42+
CsvWriter csvWriter = CsvWriter.builder()
43+
.fieldSeparator(options.delimiter())
44+
.build(csvPath)) {
45+
export(reader, csvWriter, options);
46+
}
47+
}
48+
49+
/// Export to a caller-owned [Writer]; the writer is flushed but not closed.
50+
public static void exportCsv(Path vortexPath, Writer out, ExportOptions options) throws IOException {
51+
Writer shielded = new FilterWriter(out) {
52+
@Override
53+
public void close() {
54+
// do not close the caller-owned writer
55+
}
56+
};
57+
try (VortexReader reader = VortexReader.open(vortexPath);
58+
CsvWriter csvWriter = CsvWriter.builder()
59+
.fieldSeparator(options.delimiter())
60+
.build(shielded)) {
61+
export(reader, csvWriter, options);
62+
}
63+
}
64+
65+
private static void export(VortexReader reader, CsvWriter csvWriter, ExportOptions options) throws IOException {
66+
if (!(reader.dtype() instanceof DType.Struct schema)) {
67+
throw new VortexException("only struct root dtype supported for CSV export");
68+
}
69+
List<String> colNames = schema.fieldNames();
70+
int colCount = colNames.size();
71+
72+
if (options.writeHeader()) {
73+
csvWriter.writeRecord(colNames);
74+
}
75+
76+
String[] row = new String[colCount];
77+
try (ScanIterator iter = reader.scan(ScanOptions.all())) {
78+
while (iter.hasNext()) {
79+
ScanResult chunk = iter.next();
80+
Array[] arrays = new Array[colCount];
81+
for (int c = 0; c < colCount; c++) {
82+
arrays[c] = chunk.column(colNames.get(c));
83+
}
84+
long rowCount = chunk.rowCount();
85+
for (long r = 0; r < rowCount; r++) {
86+
for (int c = 0; c < colCount; c++) {
87+
row[c] = cellValue(arrays[c], r);
88+
}
89+
csvWriter.writeRecord(row);
90+
}
91+
}
92+
}
93+
}
94+
95+
private static String cellValue(Array arr, long rowIdx) {
96+
return switch (arr) {
97+
case LongArray la -> Long.toString(la.getLong(rowIdx));
98+
case IntArray ia -> Integer.toString(ia.getInt(rowIdx));
99+
case ShortArray sa -> Short.toString(sa.getShort(rowIdx));
100+
case ByteArray ba -> Byte.toString(ba.getByte(rowIdx));
101+
case DoubleArray da -> Double.toString(da.getDouble(rowIdx));
102+
case FloatArray fa -> Float.toString(fa.getFloat(rowIdx));
103+
case BoolArray ba -> Boolean.toString(ba.getBoolean(rowIdx));
104+
case VarBinArray va -> new String(va.getBytes(rowIdx), StandardCharsets.UTF_8);
105+
default -> throw new VortexException(
106+
"unsupported array type for CSV export: " + arr.getClass().getSimpleName());
107+
};
108+
}
109+
}

‎csv/src/main/java/io/github/dfa1/vortex/csv/CsvImporter.java‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
package io.github.dfa1.vortex.csv;
2+
3+
import de.siegmar.fastcsv.reader.CsvReader;
4+
import de.siegmar.fastcsv.reader.CsvRecord;
5+
import io.github.dfa1.vortex.core.DType;
6+
import io.github.dfa1.vortex.core.PType;
7+
import io.github.dfa1.vortex.writer.VortexWriter;
8+
import io.github.dfa1.vortex.writer.WriteOptions;
9+
10+
import java.io.IOException;
11+
import java.nio.channels.FileChannel;
12+
import java.nio.file.Path;
13+
import java.nio.file.StandardOpenOption;
14+
import java.util.ArrayList;
15+
import java.util.LinkedHashMap;
16+
import java.util.List;
17+
import java.util.Map;
18+
19+
/// Parses a CSV file and writes a Vortex file.
20+
///
21+
/// Column types are inferred in priority order: long → double → boolean → utf8.
22+
/// Provide a schema via [ImportOptions#withSchema] to skip inference.
23+
/// Empty cell values are treated as 0 / false / "" for typed columns.
24+
public final class CsvImporter {
25+
26+
private CsvImporter() {
27+
}
28+
29+
public static void importCsv(Path csvPath, Path vortexPath) throws IOException {
30+
importCsv(csvPath, vortexPath, ImportOptions.defaults());
31+
}
32+
33+
public static void importCsv(Path csvPath, Path vortexPath, ImportOptions options) throws IOException {
34+
List<String[]> rows = readAllRows(csvPath, options);
35+
if (rows.isEmpty()) {
36+
throw new IllegalArgumentException("CSV file is empty");
37+
}
38+
39+
String[] headers;
40+
int dataStart;
41+
if (options.hasHeader()) {
42+
headers = rows.get(0);
43+
dataStart = 1;
44+
} else {
45+
headers = generateHeaders(rows.get(0).length);
46+
dataStart = 0;
47+
}
48+
49+
List<String[]> dataRows = rows.subList(dataStart, rows.size());
50+
int colCount = headers.length;
51+
52+
DType.Struct schema;
53+
if (options.schema() != null) {
54+
schema = options.schema();
55+
} else {
56+
schema = inferSchema(headers, dataRows, colCount);
57+
}
58+
59+
try (FileChannel channel = FileChannel.open(
60+
vortexPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE,
61+
StandardOpenOption.TRUNCATE_EXISTING);
62+
VortexWriter writer = VortexWriter.create(channel, schema, WriteOptions.defaults())) {
63+
int chunkSize = options.chunkSize();
64+
for (int start = 0; start < dataRows.size(); start += chunkSize) {
65+
int end = Math.min(start + chunkSize, dataRows.size());
66+
writer.writeChunk(buildChunk(schema, dataRows.subList(start, end)));
67+
}
68+
}
69+
}
70+
71+
private static List<String[]> readAllRows(Path path, ImportOptions options) throws IOException {
72+
List<String[]> rows = new ArrayList<>();
73+
try (CsvReader<CsvRecord> reader = CsvReader.builder()
74+
.fieldSeparator(options.delimiter())
75+
.ofCsvRecord(path)) {
76+
for (CsvRecord record : reader) {
77+
rows.add(record.getFields().toArray(String[]::new));
78+
}
79+
}
80+
return rows;
81+
}
82+
83+
private static String[] generateHeaders(int colCount) {
84+
String[] headers = new String[colCount];
85+
for (int i = 0; i < colCount; i++) {
86+
headers[i] = "col" + i;
87+
}
88+
return headers;
89+
}
90+
91+
private static DType.Struct inferSchema(String[] headers, List<String[]> rows, int colCount) {
92+
List<String> names = List.of(headers);
93+
List<DType> types = new ArrayList<>(colCount);
94+
for (int c = 0; c < colCount; c++) {
95+
types.add(inferColumnType(rows, c));
96+
}
97+
return new DType.Struct(names, types, false);
98+
}
99+
100+
private static DType inferColumnType(List<String[]> rows, int colIdx) {
101+
boolean canBeLong = true;
102+
boolean canBeDouble = true;
103+
boolean canBeBool = true;
104+
105+
for (String[] row : rows) {
106+
String val = safeGet(row, colIdx);
107+
if (val.isEmpty()) {
108+
continue;
109+
}
110+
if (canBeLong) {
111+
try {
112+
Long.parseLong(val);
113+
} catch (NumberFormatException e) {
114+
canBeLong = false;
115+
}
116+
}
117+
if (canBeDouble) {
118+
try {
119+
Double.parseDouble(val);
120+
} catch (NumberFormatException e) {
121+
canBeDouble = false;
122+
}
123+
}
124+
if (canBeBool) {
125+
if (!val.equalsIgnoreCase("true") && !val.equalsIgnoreCase("false")) {
126+
canBeBool = false;
127+
}
128+
}
129+
}
130+
131+
if (canBeLong) {
132+
return new DType.Primitive(PType.I64, false);
133+
}
134+
if (canBeDouble) {
135+
return new DType.Primitive(PType.F64, false);
136+
}
137+
if (canBeBool) {
138+
return new DType.Bool(false);
139+
}
140+
return new DType.Utf8(false);
141+
}
142+
143+
private static Map<String, Object> buildChunk(DType.Struct schema, List<String[]> rows) {
144+
int n = rows.size();
145+
Map<String, Object> chunk = new LinkedHashMap<>();
146+
for (int c = 0; c < schema.fieldNames().size(); c++) {
147+
chunk.put(schema.fieldNames().get(c), buildColumn(schema.fieldTypes().get(c), rows, c, n));
148+
}
149+
return chunk;
150+
}
151+
152+
private static Object buildColumn(DType dtype, List<String[]> rows, int colIdx, int n) {
153+
return switch (dtype) {
154+
case DType.Primitive p when p.ptype() == PType.I64 -> {
155+
long[] arr = new long[n];
156+
for (int i = 0; i < n; i++) {
157+
String v = safeGet(rows.get(i), colIdx);
158+
arr[i] = v.isEmpty() ? 0L : Long.parseLong(v);
159+
}
160+
yield arr;
161+
}
162+
case DType.Primitive p when p.ptype() == PType.F64 -> {
163+
double[] arr = new double[n];
164+
for (int i = 0; i < n; i++) {
165+
String v = safeGet(rows.get(i), colIdx);
166+
arr[i] = v.isEmpty() ? 0.0 : Double.parseDouble(v);
167+
}
168+
yield arr;
169+
}
170+
case DType.Bool ignored -> {
171+
boolean[] arr = new boolean[n];
172+
for (int i = 0; i < n; i++) {
173+
arr[i] = Boolean.parseBoolean(safeGet(rows.get(i), colIdx));
174+
}
175+
yield arr;
176+
}
177+
case DType.Utf8 ignored -> {
178+
String[] arr = new String[n];
179+
for (int i = 0; i < n; i++) {
180+
arr[i] = safeGet(rows.get(i), colIdx);
181+
}
182+
yield arr;
183+
}
184+
default -> throw new UnsupportedOperationException("unsupported dtype for CSV import: " + dtype);
185+
};
186+
}
187+
188+
private static String safeGet(String[] row, int idx) {
189+
if (idx >= row.length || row[idx] == null) {
190+
return "";
191+
}
192+
return row[idx];
193+
}
194+
}

‎csv/src/main/java/io/github/dfa1/vortex/csv/ExportOptions.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package io.github.dfa1.vortex.csv;
2+
3+
/// Options controlling Vortex → CSV export.
4+
public record ExportOptions(
5+
char delimiter,
6+
boolean writeHeader
7+
) {
8+
public static ExportOptions defaults() {
9+
return new ExportOptions(',', true);
10+
}
11+
}

‎csv/src/main/java/io/github/dfa1/vortex/csv/ImportOptions.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package io.github.dfa1.vortex.csv;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
5+
/// Options controlling CSV → Vortex import.
6+
public record ImportOptions(
7+
char delimiter,
8+
int chunkSize,
9+
boolean hasHeader,
10+
DType.Struct schema
11+
) {
12+
public static ImportOptions defaults() {
13+
return new ImportOptions(',', 65_536, true, null);
14+
}
15+
16+
/// Override the inferred schema. The struct's field names become column names;
17+
/// types control how each CSV column is parsed (positionally).
18+
public ImportOptions withSchema(DType.Struct overrideSchema) {
19+
return new ImportOptions(delimiter, chunkSize, hasHeader, overrideSchema);
20+
}
21+
}

‎csv/src/main/java/module-info.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
module io.github.dfa1.vortex.csv {
2+
requires io.github.dfa1.vortex.core;
3+
requires io.github.dfa1.vortex.reader;
4+
requires io.github.dfa1.vortex.writer;
5+
requires de.siegmar.fastcsv;
6+
7+
exports io.github.dfa1.vortex.csv;
8+
}

0 commit comments

Comments
 (0)