partition_parameters.py

"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT."""

from __future__ import annotations
from enum import Enum
import io
import pydantic
from pydantic import model_serializer
from pydantic.functional_validators import PlainValidator
from typing import IO, List, Optional, Union
from typing_extensions import Annotated, NotRequired, TypedDict
from unstructured_client import utils
from unstructured_client.types import (
    BaseModel,
    Nullable,
    OptionalNullable,
    UNSET_SENTINEL,
)
from unstructured_client.utils import (
    FieldMetadata,
    MultipartFormMetadata,
    validate_open_enum,
)


class ChunkingStrategy(str, Enum, metaclass=utils.OpenEnumMeta):
    BASIC = "basic"
    BY_PAGE = "by_page"
    BY_SIMILARITY = "by_similarity"
    BY_TITLE = "by_title"


class FilesTypedDict(TypedDict):
    content: Union[bytes, IO[bytes], io.BufferedReader]
    file_name: str
    content_type: NotRequired[str]


class Files(BaseModel):
    content: Annotated[
        Union[bytes, IO[bytes], io.BufferedReader],
        pydantic.Field(alias=""),
        FieldMetadata(multipart=MultipartFormMetadata(content=True)),
    ]

    file_name: Annotated[
        str, pydantic.Field(alias="files"), FieldMetadata(multipart=True)
    ]

    content_type: Annotated[
        Optional[str],
        pydantic.Field(alias="Content-Type"),
        FieldMetadata(multipart=True),
    ] = None


class OutputFormat(str, Enum, metaclass=utils.OpenEnumMeta):
    r"""The format of the response. Supported formats are application/json and text/csv. Default: application/json."""

    APPLICATION_JSON = "application/json"
    TEXT_CSV = "text/csv"


class Strategy(str, Enum, metaclass=utils.OpenEnumMeta):
    r"""The strategy to use for partitioning PDF/image. Options are fast, hi_res, auto. Default: hi_res"""

    FAST = "fast"
    HI_RES = "hi_res"
    AUTO = "auto"
    OCR_ONLY = "ocr_only"
    OD_ONLY = "od_only"


class PartitionParametersTypedDict(TypedDict):
    files: FilesTypedDict
    r"""The file to extract"""
    chunking_strategy: NotRequired[Nullable[ChunkingStrategy]]
    r"""Use one of the supported strategies to chunk the returned elements after partitioning. When 'chunking_strategy' is not specified, no chunking is performed and any other chunking parameters provided are ignored. Supported strategies: 'basic', 'by_page', 'by_similarity', or 'by_title'"""
    combine_under_n_chars: NotRequired[Nullable[int]]
    r"""If chunking strategy is set, combine elements until a section reaches a length of n chars. Default: 500"""
    content_type: NotRequired[Nullable[str]]
    r"""A hint about the content type to use (such as text/markdown), when there are problems processing a specific file. This value is a MIME type in the format type/subtype."""
    coordinates: NotRequired[bool]
    r"""If `True`, return coordinates for each element extracted via OCR. Default: `False`"""
    encoding: NotRequired[Nullable[str]]
    r"""The encoding method used to decode the text input. Default: utf-8"""
    extract_image_block_types: NotRequired[List[str]]
    r"""The types of elements to extract, for use in extracting image blocks as base64 encoded data stored in metadata fields."""
    gz_uncompressed_content_type: NotRequired[Nullable[str]]
    r"""If file is gzipped, use this content type after unzipping."""
    hi_res_model_name: NotRequired[Nullable[str]]
    r"""The name of the inference model used when strategy is hi_res"""
    include_orig_elements: NotRequired[Nullable[bool]]
    r"""When a chunking strategy is specified, each returned chunk will include the elements consolidated to form that chunk as `.metadata.orig_elements`. Default: true."""
    include_page_breaks: NotRequired[bool]
    r"""If true, the output will include page breaks if the filetype supports it. Default: false"""
    include_slide_notes: NotRequired[bool]
    r"""When `True`, slide notes from .ppt and .pptx files will be included in the response. Default: `True`"""
    languages: NotRequired[List[str]]
    r"""The languages present in the document, for use in partitioning and/or OCR. See the Tesseract documentation for a full list of languages."""
    max_characters: NotRequired[Nullable[int]]
    r"""If chunking strategy is set, cut off new sections after reaching a length of n chars (hard max). Default: 500"""
    multipage_sections: NotRequired[bool]
    r"""If chunking strategy is set, determines if sections can span multiple sections. Default: true"""
    new_after_n_chars: NotRequired[Nullable[int]]
    r"""If chunking strategy is set, cut off new sections after reaching a length of n chars (soft max). Default: 1500"""
    ocr_languages: NotRequired[List[str]]
    r"""Deprecated! The languages present in the document, for use in partitioning and/or OCR"""
    output_format: NotRequired[OutputFormat]
    r"""The format of the response. Supported formats are application/json and text/csv. Default: application/json."""
    overlap: NotRequired[int]
    r"""Specifies the length of a string ('tail') to be drawn from each chunk and prefixed to the next chunk as a context-preserving mechanism. By default, this only applies to split-chunks where an oversized element is divided into multiple chunks by text-splitting. Default: 0"""
    overlap_all: NotRequired[bool]
    r"""When `True`, apply overlap between 'normal' chunks formed from whole elements and not subject to text-splitting. Use this with caution as it entails a certain level of 'pollution' of otherwise clean semantic chunk boundaries. Default: False"""
    pdf_infer_table_structure: NotRequired[bool]
    r"""Deprecated! Use skip_infer_table_types to opt out of table extraction for any file type. If False and strategy=hi_res, no Table Elements will be extracted from pdf files regardless of skip_infer_table_types contents."""
    similarity_threshold: NotRequired[Nullable[float]]
    r"""A value between 0.0 and 1.0 describing the minimum similarity two elements must have to be included in the same chunk. Note that similar elements may be separated to meet chunk-size criteria; this value can only guarantees that two elements with similarity below the threshold will appear in separate chunks."""
    skip_infer_table_types: NotRequired[List[str]]
    r"""The document types that you want to skip table extraction with. Default: []"""
    split_pdf_allow_failed: NotRequired[bool]
    r"""When `split_pdf_page` is set to `True`, this parameter defines the behavior when some of the parallel requests fail. By default `split_pdf_allow_failed` is set to `False` and any failed request send to the API will make the whole process break and raise an Exception. If `split_pdf_allow_failed` is set to `True`, the errors encountered while sending parallel requests will not break the processing - the resuling list of Elements will miss the data from errored pages."""
    split_pdf_concurrency_level: NotRequired[int]
    r"""When `split_pdf_page` is set to `True`, this parameter specifies the number of workers used for sending requests when the PDF is split on the client side. It's an internal parameter for the Python client and is not sent to the backend."""
    split_pdf_page: NotRequired[bool]
    r"""This parameter determines if the PDF file should be split on the client side. It's an internal parameter for the Python client and is not sent to the backend."""
    split_pdf_page_range: NotRequired[List[int]]
    r"""When `split_pdf_page is set to `True`, this parameter selects a subset of the pdf to send to the API. The parameter is a list of 2 integers within the range [1, length_of_pdf]. A ValueError is thrown if the given range is invalid. It's an internal parameter for the Python client and is not sent to the backend."""
    starting_page_number: NotRequired[Nullable[int]]
    r"""When PDF is split into pages before sending it into the API, providing this information will allow the page number to be assigned correctly. Introduced in 1.0.27."""
    strategy: NotRequired[Strategy]
    r"""The strategy to use for partitioning PDF/image. Options are fast, hi_res, auto. Default: hi_res"""
    unique_element_ids: NotRequired[bool]
    r"""When `True`, assign UUIDs to element IDs, which guarantees their uniqueness (useful when using them as primary keys in database). Otherwise a SHA-256 of element text is used. Default: `False`"""
    xml_keep_tags: NotRequired[bool]
    r"""If `True`, will retain the XML tags in the output. Otherwise it will simply extract the text from within the tags. Only applies to XML documents."""


class PartitionParameters(BaseModel):
    files: Annotated[
        Files,
        pydantic.Field(alias=""),
        FieldMetadata(multipart=MultipartFormMetadata(file=True)),
    ]
    r"""The file to extract"""

    chunking_strategy: Annotated[
        Annotated[
            OptionalNullable[ChunkingStrategy],
            PlainValidator(validate_open_enum(False)),
        ],
        FieldMetadata(multipart=True),
    ] = None
    r"""Use one of the supported strategies to chunk the returned elements after partitioning. When 'chunking_strategy' is not specified, no chunking is performed and any other chunking parameters provided are ignored. Supported strategies: 'basic', 'by_page', 'by_similarity', or 'by_title'"""

    combine_under_n_chars: Annotated[
        OptionalNullable[int], FieldMetadata(multipart=True)
    ] = None
    r"""If chunking strategy is set, combine elements until a section reaches a length of n chars. Default: 500"""

    content_type: Annotated[OptionalNullable[str], FieldMetadata(multipart=True)] = None
    r"""A hint about the content type to use (such as text/markdown), when there are problems processing a specific file. This value is a MIME type in the format type/subtype."""

    coordinates: Annotated[Optional[bool], FieldMetadata(multipart=True)] = False
    r"""If `True`, return coordinates for each element extracted via OCR. Default: `False`"""

    encoding: Annotated[OptionalNullable[str], FieldMetadata(multipart=True)] = None
    r"""The encoding method used to decode the text input. Default: utf-8"""

    extract_image_block_types: Annotated[
        Optional[List[str]], FieldMetadata(multipart=True)
    ] = None
    r"""The types of elements to extract, for use in extracting image blocks as base64 encoded data stored in metadata fields."""

    gz_uncompressed_content_type: Annotated[
        OptionalNullable[str], FieldMetadata(multipart=True)
    ] = None
    r"""If file is gzipped, use this content type after unzipping."""

    hi_res_model_name: Annotated[
        OptionalNullable[str], FieldMetadata(multipart=True)
    ] = None
    r"""The name of the inference model used when strategy is hi_res"""

    include_orig_elements: Annotated[
        OptionalNullable[bool], FieldMetadata(multipart=True)
    ] = None
    r"""When a chunking strategy is specified, each returned chunk will include the elements consolidated to form that chunk as `.metadata.orig_elements`. Default: true."""

    include_page_breaks: Annotated[Optional[bool], FieldMetadata(multipart=True)] = (
        False
    )
    r"""If true, the output will include page breaks if the filetype supports it. Default: false"""

    include_slide_notes: Annotated[Optional[bool], FieldMetadata(multipart=True)] = True
    r"""When `True`, slide notes from .ppt and .pptx files will be included in the response. Default: `True`"""

    languages: Annotated[Optional[List[str]], FieldMetadata(multipart=True)] = None
    r"""The languages present in the document, for use in partitioning and/or OCR. See the Tesseract documentation for a full list of languages."""

    max_characters: Annotated[OptionalNullable[int], FieldMetadata(multipart=True)] = (
        None
    )
    r"""If chunking strategy is set, cut off new sections after reaching a length of n chars (hard max). Default: 500"""

    multipage_sections: Annotated[Optional[bool], FieldMetadata(multipart=True)] = True
    r"""If chunking strategy is set, determines if sections can span multiple sections. Default: true"""

    new_after_n_chars: Annotated[
        OptionalNullable[int], FieldMetadata(multipart=True)
    ] = None
    r"""If chunking strategy is set, cut off new sections after reaching a length of n chars (soft max). Default: 1500"""

    ocr_languages: Annotated[Optional[List[str]], FieldMetadata(multipart=True)] = None
    r"""Deprecated! The languages present in the document, for use in partitioning and/or OCR"""

    output_format: Annotated[
        Annotated[Optional[OutputFormat], PlainValidator(validate_open_enum(False))],
        FieldMetadata(multipart=True),
    ] = OutputFormat.APPLICATION_JSON
    r"""The format of the response. Supported formats are application/json and text/csv. Default: application/json."""

    overlap: Annotated[Optional[int], FieldMetadata(multipart=True)] = 0
    r"""Specifies the length of a string ('tail') to be drawn from each chunk and prefixed to the next chunk as a context-preserving mechanism. By default, this only applies to split-chunks where an oversized element is divided into multiple chunks by text-splitting. Default: 0"""

    overlap_all: Annotated[Optional[bool], FieldMetadata(multipart=True)] = False
    r"""When `True`, apply overlap between 'normal' chunks formed from whole elements and not subject to text-splitting. Use this with caution as it entails a certain level of 'pollution' of otherwise clean semantic chunk boundaries. Default: False"""

    pdf_infer_table_structure: Annotated[
        Optional[bool], FieldMetadata(multipart=True)
    ] = True
    r"""Deprecated! Use skip_infer_table_types to opt out of table extraction for any file type. If False and strategy=hi_res, no Table Elements will be extracted from pdf files regardless of skip_infer_table_types contents."""

    similarity_threshold: Annotated[
        OptionalNullable[float], FieldMetadata(multipart=True)
    ] = None
    r"""A value between 0.0 and 1.0 describing the minimum similarity two elements must have to be included in the same chunk. Note that similar elements may be separated to meet chunk-size criteria; this value can only guarantees that two elements with similarity below the threshold will appear in separate chunks."""

    skip_infer_table_types: Annotated[
        Optional[List[str]], FieldMetadata(multipart=True)
    ] = None
    r"""The document types that you want to skip table extraction with. Default: []"""

    split_pdf_allow_failed: Annotated[Optional[bool], FieldMetadata(multipart=True)] = (
        False
    )
    r"""When `split_pdf_page` is set to `True`, this parameter defines the behavior when some of the parallel requests fail. By default `split_pdf_allow_failed` is set to `False` and any failed request send to the API will make the whole process break and raise an Exception. If `split_pdf_allow_failed` is set to `True`, the errors encountered while sending parallel requests will not break the processing - the resuling list of Elements will miss the data from errored pages."""

    split_pdf_concurrency_level: Annotated[
        Optional[int], FieldMetadata(multipart=True)
    ] = 5
    r"""When `split_pdf_page` is set to `True`, this parameter specifies the number of workers used for sending requests when the PDF is split on the client side. It's an internal parameter for the Python client and is not sent to the backend."""

    split_pdf_page: Annotated[Optional[bool], FieldMetadata(multipart=True)] = True
    r"""This parameter determines if the PDF file should be split on the client side. It's an internal parameter for the Python client and is not sent to the backend."""

    split_pdf_page_range: Annotated[
        Optional[List[int]], FieldMetadata(multipart=True)
    ] = None
    r"""When `split_pdf_page is set to `True`, this parameter selects a subset of the pdf to send to the API. The parameter is a list of 2 integers within the range [1, length_of_pdf]. A ValueError is thrown if the given range is invalid. It's an internal parameter for the Python client and is not sent to the backend."""

    starting_page_number: Annotated[
        OptionalNullable[int], FieldMetadata(multipart=True)
    ] = None
    r"""When PDF is split into pages before sending it into the API, providing this information will allow the page number to be assigned correctly. Introduced in 1.0.27."""

    strategy: Annotated[
        Annotated[Optional[Strategy], PlainValidator(validate_open_enum(False))],
        FieldMetadata(multipart=True),
    ] = Strategy.HI_RES
    r"""The strategy to use for partitioning PDF/image. Options are fast, hi_res, auto. Default: hi_res"""

    unique_element_ids: Annotated[Optional[bool], FieldMetadata(multipart=True)] = False
    r"""When `True`, assign UUIDs to element IDs, which guarantees their uniqueness (useful when using them as primary keys in database). Otherwise a SHA-256 of element text is used. Default: `False`"""

    xml_keep_tags: Annotated[Optional[bool], FieldMetadata(multipart=True)] = False
    r"""If `True`, will retain the XML tags in the output. Otherwise it will simply extract the text from within the tags. Only applies to XML documents."""

    @model_serializer(mode="wrap")
    def serialize_model(self, handler):
        optional_fields = [
            "chunking_strategy",
            "combine_under_n_chars",
            "content_type",
            "coordinates",
            "encoding",
            "extract_image_block_types",
            "gz_uncompressed_content_type",
            "hi_res_model_name",
            "include_orig_elements",
            "include_page_breaks",
            "include_slide_notes",
            "languages",
            "max_characters",
            "multipage_sections",
            "new_after_n_chars",
            "ocr_languages",
            "output_format",
            "overlap",
            "overlap_all",
            "pdf_infer_table_structure",
            "similarity_threshold",
            "skip_infer_table_types",
            "split_pdf_allow_failed",
            "split_pdf_concurrency_level",
            "split_pdf_page",
            "split_pdf_page_range",
            "starting_page_number",
            "strategy",
            "unique_element_ids",
            "xml_keep_tags",
        ]
        nullable_fields = [
            "chunking_strategy",
            "combine_under_n_chars",
            "content_type",
            "encoding",
            "gz_uncompressed_content_type",
            "hi_res_model_name",
            "include_orig_elements",
            "max_characters",
            "new_after_n_chars",
            "similarity_threshold",
            "starting_page_number",
        ]
        null_default_fields = [
            "chunking_strategy",
            "combine_under_n_chars",
            "content_type",
            "encoding",
            "gz_uncompressed_content_type",
            "hi_res_model_name",
            "include_orig_elements",
            "max_characters",
            "new_after_n_chars",
            "similarity_threshold",
            "starting_page_number",
        ]

        serialized = handler(self)

        m = {}

        for n, f in self.model_fields.items():
            k = f.alias or n
            val = serialized.get(k)
            serialized.pop(k, None)

            optional_nullable = k in optional_fields and k in nullable_fields
            is_set = (
                self.__pydantic_fields_set__.intersection({n})
                or k in null_default_fields
            )  # pylint: disable=no-member

            if val is not None and val != UNSET_SENTINEL:
                m[k] = val
            elif val != UNSET_SENTINEL and (
                not k in optional_fields or (optional_nullable and is_set)
            ):
                m[k] = val

        return m