From b096272f753958ca13d9302f2f0d0f5cb5e142b0 Mon Sep 17 00:00:00 2001 From: katsu560 Date: Wed, 26 Jun 2024 04:54:01 +0900 Subject: [PATCH 1/8] sync ggml --- gguf-py/scripts/gguf-addfile.py | 178 ++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 gguf-py/scripts/gguf-addfile.py diff --git a/gguf-py/scripts/gguf-addfile.py b/gguf-py/scripts/gguf-addfile.py new file mode 100644 index 0000000000000..1dd00f3aa8342 --- /dev/null +++ b/gguf-py/scripts/gguf-addfile.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# gguf-addfile.py srcfile dstfile addfiles ... + +from __future__ import annotations + +import logging +import argparse +import os +import sys +from pathlib import Path +from typing import Any +#from typing import Any, Literal, NamedTuple, TypeVar, Union + +import numpy as np +import numpy.typing as npt + +# Necessary to load the local gguf package +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): + sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFReader, GGUFWriter, ReaderField, GGMLQuantizationType, GGUFEndian, GGUFValueType, Keys # noqa: E402 + +logger = logging.getLogger("gguf-addfile") + + +def get_file_host_endian(reader: GGUFReader) -> tuple[str, str]: + host_endian = 'LITTLE' if np.uint32(1) == np.uint32(1).newbyteorder("<") else 'BIG' + if reader.byte_order == 'S': + file_endian = 'BIG' if host_endian == 'LITTLE' else 'LITTLE' + else: + file_endian = host_endian + return (host_endian, file_endian) + + +def get_byteorder(reader: GGUFReader) -> GGUFEndian: + if np.uint32(1) == np.uint32(1).newbyteorder("<"): + # Host is little endian + host_endian = GGUFEndian.LITTLE + swapped_endian = GGUFEndian.BIG + else: + # Sorry PDP or other weird systems that don't use BE or LE. + host_endian = GGUFEndian.BIG + swapped_endian = GGUFEndian.LITTLE + + if reader.byte_order == "S": + return swapped_endian + else: + return host_endian + + +def decode_field(field: ReaderField) -> Any: + if field and field.types: + main_type = field.types[0] + + if main_type == GGUFValueType.ARRAY: + sub_type = field.types[-1] + + if sub_type == GGUFValueType.STRING: + return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data] + else: + return [pv for idx in field.data for pv in field.parts[idx].tolist()] + if main_type == GGUFValueType.STRING: + return str(bytes(field.parts[-1]), encoding='utf8') + else: + return field.parts[-1][0] + + return None + + +def get_field_data(reader: GGUFReader, key: str) -> Any: + field = reader.get_field(key) + + return decode_field(field) + + +def copy_with_filename(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, filename: str[Any]) -> None: + logger.debug(f'copy_with_filename: {filename}') #debug + val = filename + for field in reader.fields.values(): + # Suppress virtual fields and fields written by GGUFWriter + if field.name == Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): + logger.debug(f'Suppressing {field.name}') + continue + + # Copy existed fields except 'embedded_files' + if not field.name == Keys.EMBEDDED_FILES: + cur_val = decode_field(field) + writer.add_key(field.name) + writer.add_val(cur_val, field.types[0]) + logger.debug(f'Copying {field.name}') + continue + + # Update embedded_files + val = decode_field(field) + for path in filename: + logger.debug(f'Adding {field.name}: {path}') + val.append(path) + + # Add filenames to kv + logger.info(f'* Modifying {Keys.EMBEDDED_FILES} to {val}') + writer.add_array(Keys.EMBEDDED_FILES, val) + + for tensor in reader.tensors: + # Dimensions are written in reverse order, so flip them first + shape = np.flipud(tensor.shape) + writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + + # Add file info as tensor_info + for path in filename: + logger.debug(f'Adding tensor_info {path}') + with open(path, "rb") as f: + data = f.read() + data_len = len(data) + dims = [data_len] + raw_dtype = GGMLQuantizationType.I8 + writer.add_tensor_info(path, dims, np.float16, data_len, raw_dtype) + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_ti_data_to_file() + + for tensor in reader.tensors: + writer.write_tensor_data(tensor.data) + + # Write file body as tensor data + for path in filename: + logger.debug(f'Adding tensor data {path}') + with open(path, "rb") as f: + data = f.read() + data_len = len(data) + # write data with padding + writer.write_data(data) + + writer.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Add files to GGUF file metadata") + parser.add_argument("input", type=str, help="GGUF format model input filename") + parser.add_argument("output", type=str, help="GGUF format model output filename") + parser.add_argument("addfiles", type=str, nargs='+', help="add filenames ...") + parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") + parser.add_argument("--verbose", action="store_true", help="Increase output verbosity") + args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + logger.info(f'* Loading: {args.input}') + reader = GGUFReader(args.input, 'r') + arch = get_field_data(reader, Keys.General.ARCHITECTURE) + endianess = get_byteorder(reader) + + if os.path.isfile(args.output) and not args.force: + logger.warning('*** Warning *** Warning *** Warning **') + logger.warning(f'* The "{args.output}" GGUF file already exists, it will be overwritten!') + logger.warning('* Enter exactly YES if you are positive you want to proceed:') + response = input('YES, I am sure> ') + if response != 'YES': + logger.info("You didn't enter YES. Okay then, see ya!") + sys.exit(0) + + logger.info(f'* Writing: {args.output}') + writer = GGUFWriter(args.output, arch=arch, endianess=endianess) + + alignment = get_field_data(reader, Keys.General.ALIGNMENT) + if alignment is not None: + logger.debug(f'Setting custom alignment: {alignment}') + writer.data_alignment = alignment + + if args.addfiles is not None: + filename = [] + for path in args.addfiles: + filename.append(path) + logger.info(f'* Adding: {path}') + copy_with_filename(reader, writer, filename) + + +if __name__ == '__main__': + main() From e4562042a40ac1b4e203708e4860e1d717928e85 Mon Sep 17 00:00:00 2001 From: katsu560 Date: Wed, 26 Jun 2024 04:55:13 +0900 Subject: [PATCH 2/8] sync ggml --- gguf-py/gguf/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 222a2d137b08f..c839f7e97b7f1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -18,6 +18,8 @@ class Keys: + EMBEDDED_FILES = "embedded_files" + class General: ARCHITECTURE = "general.architecture" QUANTIZATION_VERSION = "general.quantization_version" From 562d4f2de43a4a63cbf38df4ae46d82c0383f2ba Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 20 Jul 2024 18:41:13 +0900 Subject: [PATCH 3/8] delete commented line --- gguf-py/scripts/gguf-addfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gguf-py/scripts/gguf-addfile.py b/gguf-py/scripts/gguf-addfile.py index 1dd00f3aa8342..4f804b753e571 100644 --- a/gguf-py/scripts/gguf-addfile.py +++ b/gguf-py/scripts/gguf-addfile.py @@ -9,7 +9,6 @@ import sys from pathlib import Path from typing import Any -#from typing import Any, Literal, NamedTuple, TypeVar, Union import numpy as np import numpy.typing as npt From 2bad597a5a6d137bb4f4e1ff36d7d3ed4f6666a0 Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 27 Jul 2024 18:54:45 +0900 Subject: [PATCH 4/8] rename to gguf_add_file.py --- gguf-py/scripts/{gguf-addfile.py => gguf_add_file.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gguf-py/scripts/{gguf-addfile.py => gguf_add_file.py} (100%) diff --git a/gguf-py/scripts/gguf-addfile.py b/gguf-py/scripts/gguf_add_file.py similarity index 100% rename from gguf-py/scripts/gguf-addfile.py rename to gguf-py/scripts/gguf_add_file.py From 48856e1251dbb24b6484f1f18f640f7a5a35bc3b Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 27 Jul 2024 18:57:47 +0900 Subject: [PATCH 5/8] update comment --- gguf-py/scripts/gguf_add_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/scripts/gguf_add_file.py b/gguf-py/scripts/gguf_add_file.py index 4f804b753e571..76f937058ec87 100644 --- a/gguf-py/scripts/gguf_add_file.py +++ b/gguf-py/scripts/gguf_add_file.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# gguf-addfile.py srcfile dstfile addfiles ... +# gguf_add_file.py srcfile dstfile addfiles ... from __future__ import annotations From a4289e2aab0fd41318776d1bcd20075fe23e929b Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 10 Aug 2024 19:57:20 +0900 Subject: [PATCH 6/8] add EMBEDDED --- gguf-py/gguf/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 89efe0c800964..4ba3e50d8f37d 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -18,6 +18,8 @@ class Keys: + EMBEDDED_FILES = "embedded_files" + class General: TYPE = "general.type" ARCHITECTURE = "general.architecture" From 01b463f0e81f273f49ad50f51511686740913f18 Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 10 Aug 2024 19:57:37 +0900 Subject: [PATCH 7/8] add write_data --- gguf-py/gguf/gguf_writer.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 76385a82872c9..6f14984750a50 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -391,6 +391,23 @@ def write_padding(self, fp: IO[bytes], n: int, align: int | None = None) -> None if pad != 0: fp.write(bytes([0] * pad)) + def write_data(self, data: bytes[Any]) -> None: + if self.state is not WriterState.TI_DATA and self.state is not WriterState.WEIGHTS: + raise ValueError(f'Expected output file to contain tensor info or weights, got {self.state}') + assert self.fout is not None + + file_id = -1 + for i, tensors in enumerate(self.tensors): + if len(tensors) > 0: + file_id = i + break + + fout = self.fout[file_id] + + self.write_padding(fout, fout.tell()) + fout.write(data) + self.write_padding(fout, len(data)) + def write_tensor_data(self, tensor: np.ndarray[Any, Any]) -> None: if self.state is not WriterState.TI_DATA and self.state is not WriterState.WEIGHTS: raise ValueError(f'Expected output file to contain tensor info or weights, got {self.state}') From 287f7db329099cf06142333009a74b97353e3be9 Mon Sep 17 00:00:00 2001 From: katsu560 Date: Sat, 10 Aug 2024 19:58:25 +0900 Subject: [PATCH 8/8] add gguf_add_file.py --- gguf-py/scripts/gguf_add_file.py | 176 +++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 gguf-py/scripts/gguf_add_file.py diff --git a/gguf-py/scripts/gguf_add_file.py b/gguf-py/scripts/gguf_add_file.py new file mode 100644 index 0000000000000..861b8d8584f95 --- /dev/null +++ b/gguf-py/scripts/gguf_add_file.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# gguf_add_file.py srcfile dstfile addfiles ... + +from __future__ import annotations + +import logging +import argparse +import os +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +# Necessary to load the local gguf package +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists(): + sys.path.insert(0, str(Path(__file__).parent.parent)) + +from gguf import GGUFReader, GGUFWriter, ReaderField, GGMLQuantizationType, GGUFEndian, GGUFValueType, Keys # noqa: E402 + +logger = logging.getLogger("gguf_add_file") + + +def get_file_host_endian(reader: GGUFReader) -> tuple[str, str]: + host_endian = 'LITTLE' if np.uint32(1) == np.uint32(1).newbyteorder("<") else 'BIG' + if reader.byte_order == 'S': + file_endian = 'BIG' if host_endian == 'LITTLE' else 'LITTLE' + else: + file_endian = host_endian + return (host_endian, file_endian) + + +def get_byteorder(reader: GGUFReader) -> GGUFEndian: + if np.uint32(1) == np.uint32(1).newbyteorder("<"): + # Host is little endian + host_endian = GGUFEndian.LITTLE + swapped_endian = GGUFEndian.BIG + else: + # Sorry PDP or other weird systems that don't use BE or LE. + host_endian = GGUFEndian.BIG + swapped_endian = GGUFEndian.LITTLE + + if reader.byte_order == "S": + return swapped_endian + else: + return host_endian + + +def decode_field(field: ReaderField) -> Any: + if field and field.types: + main_type = field.types[0] + + if main_type == GGUFValueType.ARRAY: + sub_type = field.types[-1] + + if sub_type == GGUFValueType.STRING: + return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data] + else: + return [pv for idx in field.data for pv in field.parts[idx].tolist()] + if main_type == GGUFValueType.STRING: + return str(bytes(field.parts[-1]), encoding='utf8') + else: + return field.parts[-1][0] + + return None + + +def get_field_data(reader: GGUFReader, key: str) -> Any: + field = reader.get_field(key) + + return decode_field(field) + + +def copy_with_filename(reader: GGUFReader, writer: GGUFWriter, filename: str[Any]) -> None: + logger.debug(f'copy_with_filename: {filename}') # debug + val = filename + for field in reader.fields.values(): + # Suppress virtual fields and fields written by GGUFWriter + if field.name == Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'): + logger.debug(f'Suppressing {field.name}') + continue + + # Copy existed fields except 'embedded_files' + if not field.name == Keys.EMBEDDED_FILES: + cur_val = decode_field(field) + writer.add_key(field.name) + writer.add_val(cur_val, field.types[0]) + logger.debug(f'Copying {field.name}') + continue + + # Update embedded_files + val = decode_field(field) + for path in filename: + logger.debug(f'Adding {field.name}: {path}') + val.append(path) + + # Add filenames to kv + logger.info(f'* Modifying {Keys.EMBEDDED_FILES} to {val}') + writer.add_array(Keys.EMBEDDED_FILES, val) + + for tensor in reader.tensors: + # Dimensions are written in reverse order, so flip them first + shape = np.flipud(tensor.shape) + writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + + # Add file info as tensor_info + for path in filename: + logger.debug(f'Adding tensor_info {path}') + with open(path, "rb") as f: + data = f.read() + data_len = len(data) + dims = [data_len] + raw_dtype = GGMLQuantizationType.I8 + writer.add_tensor_info(path, dims, np.float16, data_len, raw_dtype) + + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_ti_data_to_file() + + for tensor in reader.tensors: + writer.write_tensor_data(tensor.data) + + # Write file body as tensor data + for path in filename: + logger.debug(f'Adding tensor data {path}') + with open(path, "rb") as f: + data = f.read() + data_len = len(data) + # write data with padding + writer.write_data(data) + + writer.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Add files to GGUF file metadata") + parser.add_argument("input", type=str, help="GGUF format model input filename") + parser.add_argument("output", type=str, help="GGUF format model output filename") + parser.add_argument("addfiles", type=str, nargs='+', help="add filenames ...") + parser.add_argument("--force", action="store_true", help="Bypass warnings without confirmation") + parser.add_argument("--verbose", action="store_true", help="Increase output verbosity") + args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + logger.info(f'* Loading: {args.input}') + reader = GGUFReader(args.input, 'r') + arch = get_field_data(reader, Keys.General.ARCHITECTURE) + endianess = get_byteorder(reader) + + if os.path.isfile(args.output) and not args.force: + logger.warning('*** Warning *** Warning *** Warning **') + logger.warning(f'* The "{args.output}" GGUF file already exists, it will be overwritten!') + logger.warning('* Enter exactly YES if you are positive you want to proceed:') + response = input('YES, I am sure> ') + if response != 'YES': + logger.info("You didn't enter YES. Okay then, see ya!") + sys.exit(0) + + logger.info(f'* Writing: {args.output}') + writer = GGUFWriter(args.output, arch=arch, endianess=endianess) + + alignment = get_field_data(reader, Keys.General.ALIGNMENT) + if alignment is not None: + logger.debug(f'Setting custom alignment: {alignment}') + writer.data_alignment = alignment + + if args.addfiles is not None: + filename = [] + for path in args.addfiles: + filename.append(path) + logger.info(f'* Adding: {path}') + copy_with_filename(reader, writer, filename) + + +if __name__ == '__main__': + main()