From d42cb24f3f2e005e1b59101017ed478516ea47fc Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 30 Jul 2024 11:38:33 -0700 Subject: [PATCH] Docs: Add destinations API reference docs (#314) --- airbyte/destinations/__init__.py | 74 +++++++++++++++++++++++++++++++- airbyte/destinations/base.py | 5 ++- airbyte/destinations/util.py | 5 ++- 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/airbyte/destinations/__init__.py b/airbyte/destinations/__init__.py index fc7a4ca6..c35d1786 100644 --- a/airbyte/destinations/__init__.py +++ b/airbyte/destinations/__init__.py @@ -1,5 +1,77 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -"""Destinations module.""" +"""Destinations module. + +This module contains classes and methods for interacting with Airbyte destinations. You can use this +module to create custom destinations, or to interact with existing destinations. + +## Getting Started + +To get started with destinations, you can use the `get_destination()` method to create a destination +object. This method takes a destination name and configuration, and returns a destination object +that you can use to write data to the destination. + +```python +import airbyte as ab + +my_destination = ab.get_destination( + "destination-foo", + config={"api_key": "my_api_key"}, + docker_image=True, +) +``` + +## Writing Data to a Destination + +To write data to a destination, you can use the `Destination.write()` method. This method +takes either a `airbyte.Source` or `airbyte.ReadResult` object. + +## Writing to a destination from a source + +To write directly from a source, simply pass the source object to the `Destination.write()` method: + +```python +my_source = get_source(...) +my_destination = get_destination(...) +my_destination.write(source_faker) +``` + +## Writing from a read result: + +To write from a read result, you can use the following pattern. First, read data from the source, +then write the data to the destination, using the `ReadResult` object as a buffer between the source +and destination: + +```python +# First read data from the source: +my_source = get_source(...) +read_result = my_source.read(...) + +# Optionally, you can validate data before writing it: +# ...misc validation code here... + +# Then write the data to the destination: +my_destination.write(read_result) +``` + +## Using Docker and Python-based Connectors + +By default, the `get_destination()` method will look for a Python-based connector. If you want to +use a Docker-based connector, you can set the `docker_image` parameter to `True`: + +```python +my_destination = ab.get_destination( + "destination-foo", + config={"api_key": "my_api_key"}, + docker_image=True, +) +``` + +**Note:** Unlike source connectors, most destination connectors are written in Java, and for this +reason are only available as Docker-based connectors. If you need to load to a SQL database and your +runtime does not support docker, you may want to use the `airbyte.caches` module to load data to +a SQL cache. Caches are mostly identical to destinations in behavior, and are implemented internally +to PyAirbyte so they can run anywhere that PyAirbyte can run. +""" from __future__ import annotations diff --git a/airbyte/destinations/base.py b/airbyte/destinations/base.py index 3f18ac57..a61522c1 100644 --- a/airbyte/destinations/base.py +++ b/airbyte/destinations/base.py @@ -1,5 +1,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -"""Destination base classes.""" +"""Destination base classes. + +For usage examples, see the `airbyte.destinations` module documentation. +""" from __future__ import annotations diff --git a/airbyte/destinations/util.py b/airbyte/destinations/util.py index f80cd46e..59b1354e 100644 --- a/airbyte/destinations/util.py +++ b/airbyte/destinations/util.py @@ -1,5 +1,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -"""Destination utilities.""" +"""Destination utilities. + +For usage examples, see the `airbyte.destinations` module documentation. +""" from __future__ import annotations