Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions bench/benchmark_ldbc.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ def run_one(
str(REPO_ROOT / "bench" / "run_one.py"),
"--source-dir",
str(source_dir),
"--output-db",
str(out_dir / f"{backend}.duckdb"),
"--output-dir",
str(out_dir / f"{backend}-csr"),
"--backend",
backend,
*(["--memory-limit", memory_limit] if memory_limit else []),
Expand Down
4 changes: 2 additions & 2 deletions bench/compare_memgraph.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ def main() -> None:
str(REPO_ROOT / "bench" / "run_one.py"),
"--source-dir",
args.source_dir,
"--output-db",
str(run_dir / "out.duckdb"),
"--output-dir",
str(run_dir / "out-csr"),
"--backend",
"pyarrow",
*(["--memory-limit", args.memory_limit] if args.memory_limit else []),
Expand Down
4 changes: 2 additions & 2 deletions bench/run_one.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def _max_rss_mib() -> float:
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--source-dir", required=True)
ap.add_argument("--output-db", required=True)
ap.add_argument("--output-dir", required=True)
ap.add_argument(
"--backend", required=True, choices=["pyarrow", "duckdb", "datafusion"]
)
Expand All @@ -39,7 +39,7 @@ def main() -> None:
start = time.perf_counter()
results = convert_parquet_dir_to_csr(
source_dir=args.source_dir,
output_db=args.output_db,
output_dir=args.output_dir,
backend=args.backend,
memory_limit=args.memory_limit,
)
Expand Down
9 changes: 7 additions & 2 deletions icebug_format/_convert_datafusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from icebug_format.convert_parquet import (
ICEBUG_DISK_VERSION,
_write_icebug_parquet,
csr_rel_name,
parse_size_to_bytes,
resolve_rel_column_names,
)
Expand Down Expand Up @@ -342,7 +343,9 @@ def convert_graph(
f"FROM relations ORDER BY csr_source, csr_target"
)
_stream_to_parquet(
ctx.sql(idx_sql), out_dir / f"indices_{name}.parquet", target_to_uint64=True
ctx.sql(idx_sql),
out_dir / f"indices_{csr_rel_name(name)}.parquet",
target_to_uint64=True,
)

# indptr: degrees from a streaming GROUP BY, then histogram + prefix sum.
Expand All @@ -357,5 +360,7 @@ def convert_graph(
out_dir / f"nodes_{name}.parquet",
)
_write_icebug_parquet(
indptr, out_dir / f"indptr_{name}.parquet", compression=_COMPRESSION
indptr,
out_dir / f"indptr_{csr_rel_name(name)}.parquet",
compression=_COMPRESSION,
)
6 changes: 3 additions & 3 deletions icebug_format/_convert_duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import pyarrow as pa
import pyarrow.parquet as pq

from icebug_format.convert_parquet import resolve_rel_column_names
from icebug_format.convert_parquet import csr_rel_name, resolve_rel_column_names


def _q(name: str) -> str:
Expand Down Expand Up @@ -139,10 +139,10 @@ def convert_graph(
con, "vertices", out_dir / f"nodes_{name}.parquet"
)
_write_parquet_with_icebug_metadata(
con, "indices", out_dir / f"indices_{name}.parquet"
con, "indices", out_dir / f"indices_{csr_rel_name(name)}.parquet"
)
_write_parquet_with_icebug_metadata(
con, "indptr", out_dir / f"indptr_{name}.parquet"
con, "indptr", out_dir / f"indptr_{csr_rel_name(name)}.parquet"
)
finally:
con.close()
Expand Down
5 changes: 3 additions & 2 deletions icebug_format/_convert_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from icebug_format.convert_parquet import (
_build_indptr,
_write_icebug_parquet,
csr_rel_name,
resolve_rel_column_names,
)

Expand Down Expand Up @@ -344,5 +345,5 @@ def convert_graph(
indices, indptr = _finalize_csr(rel, prop_cols, add_reverse_edges, n_nodes)

_write_icebug_parquet(vtable, out_dir / f"nodes_{name}.parquet")
_write_icebug_parquet(indices, out_dir / f"indices_{name}.parquet")
_write_icebug_parquet(indptr, out_dir / f"indptr_{name}.parquet")
_write_icebug_parquet(indices, out_dir / f"indices_{csr_rel_name(name)}.parquet")
_write_icebug_parquet(indptr, out_dir / f"indptr_{csr_rel_name(name)}.parquet")
69 changes: 43 additions & 26 deletions icebug_format/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,30 @@ def duckdb_type_to_cypher_type(duckdb_type: str) -> str:
return type_map.get(base_type, "STRING")


def get_node_display_name(table_name: str) -> str:
"""Derive the NODE table name used in schema.cypher from a DuckDB table."""
if table_name == "nodes":
return "nodes"
elif table_name.startswith("nodes_"):
return table_name[6:].lower() # Remove "nodes_" prefix and lowercase
return table_name.lower()


def get_edge_display_name(table_name: str) -> str:
"""Derive the REL table name used in schema.cypher from a DuckDB table.

The ``_rel`` suffix keeps the REL table distinct from a same-named NODE
table. LadybugDB derives the ``indices_<rel_name>.parquet`` /
``indptr_<rel_name>.parquet`` filenames from this name, so the parquet
export must use the same name.
"""
if table_name == "edges":
return "edges"
elif table_name.startswith("edges_"):
return table_name[6:].lower() # Remove "edges_" prefix and lowercase
return table_name.lower() + "_rel"


def generate_schema_cypher(
con,
csr_table_name: str,
Expand All @@ -212,22 +236,6 @@ def generate_schema_cypher(
"""
lines = []

# Helper to derive display name from table name (lowercase)
# nodes => nodes, nodes_person => person, nodes_foo => foo
def get_node_display_name(table_name: str) -> str:
if table_name == "nodes":
return "nodes"
elif table_name.startswith("nodes_"):
return table_name[6:].lower() # Remove "nodes_" prefix and lowercase
return table_name.lower()

def get_edge_display_name(table_name: str) -> str:
if table_name == "edges":
return "edges"
elif table_name.startswith("edges_"):
return table_name[6:].lower() # Remove "edges_" prefix and lowercase
return table_name.lower()

# Build mapping of original table names to display names
node_display_names = {nt: get_node_display_name(nt) for nt in node_tables}

Expand Down Expand Up @@ -352,20 +360,23 @@ def get_display_name(table_name: str, prefix: str) -> str:
_write_parquet_with_icebug_metadata(con, csr_node_table, parquet_file)
print(f" Exported: {csr_node_table} -> {parquet_file.name}")

# Export edge tables: indices_<edge_name>.parquet, indptr_<edge_name>.parquet
# Export edge tables: indices_<rel_name>.parquet, indptr_<rel_name>.parquet
# (rel_name == the REL table name emitted in schema.cypher, so LadybugDB can
# map the schema TABLE back to its parquet file)
for edge_table in edge_tables:
edge_name = (
edge_table[6:].lower()
if edge_table.startswith("edges_")
else edge_table.lower()
)
rel_name = get_edge_display_name(edge_table)
indices_table = f"{csr_table_name}_indices_{edge_name}"
indices_file = parquet_dir / f"indices_{edge_name}.parquet"
indices_file = parquet_dir / f"indices_{rel_name}.parquet"
_write_parquet_with_icebug_metadata(con, indices_table, indices_file)
print(f" Exported: {indices_table} -> {indices_file.name}")

indptr_table = f"{csr_table_name}_indptr_{edge_name}"
indptr_file = parquet_dir / f"indptr_{edge_name}.parquet"
indptr_file = parquet_dir / f"indptr_{rel_name}.parquet"
_write_parquet_with_icebug_metadata(con, indptr_table, indptr_file)
print(f" Exported: {indptr_table} -> {indptr_file.name}")

Expand All @@ -377,7 +388,7 @@ def get_display_name(table_name: str, prefix: str) -> str:
edge_tables,
parquet_dir,
edge_relationships,
node_type_to_table
node_type_to_table,
)
schema_file = parquet_dir / "schema.cypher"
schema_file.write_text(schema_cypher)
Expand Down Expand Up @@ -731,6 +742,12 @@ def main():
help="Conversion backend for --source-dir input "
"(default: auto-detect an installed SQL engine, else pyarrow)",
)
parser.add_argument(
"--output-dir",
type=str,
help="Output directory for icebug-disk Parquet files (--source-dir only; "
"default: <source-dir>-csr)",
)
parser.add_argument(
"--output-db",
type=str,
Expand Down Expand Up @@ -772,7 +789,7 @@ def main():
"--storage",
type=str,
default=None,
help="Storage path for schema.cypher (default: output_db path without .duckdb extension)",
help="Storage path recorded in schema.cypher (default: derived from the output path)",
)
parser.add_argument(
"--schema",
Expand Down Expand Up @@ -807,15 +824,15 @@ def main():
print(f"Add reverse edges: {args.add_reverse_edges}")
print(f"DuckDB/DataFusion memory limit: {args.memory_limit}")

output_db = args.output_db or str(
Path(args.source_dir).parent / f"{Path(args.source_dir).name}_csr.duckdb"
output_dir = args.output_dir or str(
Path(args.source_dir).parent / f"{Path(args.source_dir).name}-csr"
)
storage_path = args.storage or f"./{Path(output_db).stem}"
print(f"Output directory: {Path(output_db).parent / Path(output_db).stem}")
storage_path = args.storage or f"./{Path(output_dir).name}"
print(f"Output directory: {output_dir}")

results = convert_parquet_dir_to_csr(
source_dir=args.source_dir,
output_db=output_db,
output_dir=output_dir,
backend=args.backend,
add_reverse_edges=args.add_reverse_edges,
memory_limit=args.memory_limit,
Expand Down
44 changes: 28 additions & 16 deletions icebug_format/convert_parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
as :mod:`icebug_format.cli` for DuckDB sources:

- ``nodes_<name>.parquet`` vertex table sorted by primary key
- ``indices_<name>.parquet`` dense target id per edge, sorted by (source, target)
- ``indptr_<name>.parquet`` CSR row pointers (``N + 1`` entries)
- ``indices_<name>_rel.parquet`` dense target id per edge, sorted by (source, target)
- ``indptr_<name>_rel.parquet`` CSR row pointers (``N + 1`` entries)
- ``schema.cypher`` LadybugDB mount schema

Three interchangeable backends are provided:
Expand Down Expand Up @@ -258,14 +258,27 @@ def _arrow_type_to_cypher(t: pa.DataType) -> str:
return "STRING"


def csr_rel_name(graph_name: str) -> str:
"""Return the REL table name for a homogeneous CSR graph.

LadybugDB derives the parquet filenames from the schema table names, so the
REL table (``<name>_rel``) must match the ``indices_<name>_rel.parquet`` /
``indptr_<name>_rel.parquet`` files written by the backends. The suffix
keeps the REL table distinct from the NODE table, whose name is the graph
name itself.
"""
return f"{graph_name}_rel"


def _generate_schema_cypher(graph_name: str, output_dir: Path, storage: str) -> str:
"""
Build schema.cypher for one graph from the parquet files just written.

The graph is homogeneous: both edge endpoints use the single vertex table.
"""
node_pq = output_dir / f"nodes_{graph_name}.parquet"
indices_pq = output_dir / f"indices_{graph_name}.parquet"
rel_name = csr_rel_name(graph_name)
indices_pq = output_dir / f"indices_{rel_name}.parquet"
node_schema = pq.ParquetFile(node_pq).schema_arrow
indices_schema = pq.ParquetFile(indices_pq).schema_arrow

Expand All @@ -284,7 +297,7 @@ def _generate_schema_cypher(graph_name: str, output_dir: Path, storage: str) ->
]
props_str = (", " + ", ".join(props)) if props else ""
lines.append(
f"CREATE REL TABLE {graph_name}(FROM {graph_name} TO {graph_name}{props_str}) "
f"CREATE REL TABLE {rel_name}(FROM {graph_name} TO {graph_name}{props_str}) "
f"WITH (storage = '{storage}', format = 'icebug-disk');"
)
return "\n".join(lines) + "\n"
Expand Down Expand Up @@ -354,7 +367,7 @@ def _select_backend(backend: str, memory_limit: str | None = None) -> Callable:

def convert_parquet_dir_to_csr(
source_dir: str | Path,
output_db: str | Path | None = None,
output_dir: str | Path | None = None,
graph_name: str | None = None,
backend: str = "auto",
add_reverse_edges: bool = False,
Expand All @@ -367,9 +380,9 @@ def convert_parquet_dir_to_csr(
Args:
source_dir: Directory containing ``<name>-v.parquet``/``<name>-e.parquet``
pairs (and/or ``vertex.parquet``/``edge.parquet``).
output_db: Output database path used only as a naming convention; the
Parquet files are written to a sibling directory named after its
stem. Defaults to ``<parent>/<source_dir.name>_csr.duckdb``.
output_dir: Output directory for the icebug-disk Parquet files. When
multiple graphs are discovered each one is written to a subdirectory
``<output_dir>/<graph_name>``. Defaults to ``<source_dir>-csr``.
graph_name: Optional filter; when omitted, every discovered graph is
converted.
backend: One of ``auto``, ``pyarrow``, ``duckdb``, ``datafusion``.
Expand Down Expand Up @@ -398,18 +411,17 @@ def convert_parquet_dir_to_csr(
f"{', '.join(g['name'] for g in graphs)}"
)

output_db = (
Path(output_db)
if output_db
else Path(source_dir).parent / f"{Path(source_dir).name}_csr.duckdb"
output_dir = (
Path(output_dir)
if output_dir
else Path(source_dir).parent / f"{Path(source_dir).name}-csr"
)
output_base = Path(output_db).parent / Path(output_db).stem
if len(graphs) == 1:
out_dirs = [output_base]
out_dirs = [output_dir]
else:
out_dirs = [output_base / g["name"] for g in graphs]
out_dirs = [output_dir / g["name"] for g in graphs]

storage_path = storage if storage is not None else f"./{Path(output_db).stem}"
storage_path = storage if storage is not None else f"./{output_dir.name}"

convert = _select_backend(backend, memory_limit)
results: list[dict] = []
Expand Down
Loading
Loading