Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/dataset.jl
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,8 @@ struct DatasetPartitions
end
end

Base.IteratorSize(::Type{DatasetPartitions}) = Base.SizeUnknown()

function iterated_partition(partitions::DatasetPartitions, cursor)
partition = nothing
walker, root, files, fileidx, step = cursor
Expand Down
1 change: 1 addition & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ const julia_parcompat = joinpath(artifact"julia_parcompat", readdir(artifact"jul
include("test_cursors.jl")
include("test_writer.jl")
include("test_dataset_types.jl")
include("test_dataset_partitions.jl")
end
36 changes: 36 additions & 0 deletions test/test_dataset_partitions.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
using Parquet, Tables, Test

@testset "Collect dataset partitions" begin
fixture = joinpath(@__DIR__, "booltest", "alltypes_plain.snappy.parquet")
mktempdir() do path
for name in ("a.parquet", "b.parquet", "_common_metadata", "_metadata")
cp(fixture, joinpath(path, name))
end
for factory in (Parquet.Dataset, Parquet.read_parquet), (filter, expected) in (
(p -> true, 2),
(p -> endswith(p, "a.parquet"), 1),
(p -> false, 0),
)
dataset = factory(path; filter=filter)
try
nmanual = 0
for partition in Tables.partitions(dataset)
nmanual += 1
close(partition)
end
@test nmanual == expected
partitions = collect(Tables.partitions(dataset))
try
@test partitions isa AbstractVector
@test length(partitions) == expected
@test all(p -> p isa Parquet.Table, partitions)
@test all(p -> Tables.getcolumn(p, :id) == Int32[6, 7], partitions)
finally
foreach(close, partitions)
end
finally
close(dataset)
end
end
end
end
Loading