Initial commit

Author: raylu

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+/build/
+/dist/
+/sqlalchemy_batch_inserts.egg-info/

Dockerfile

@@ -0,0 +1,6 @@
+FROM postgres:9.6.9
+
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip \
+  && rm -r /var/lib/apt/lists/*
+RUN pip3 install --quiet psycopg2-binary sqlalchemy

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Benchling
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,67 @@
+Benchling uses SQLAlchemy and psycopg2 to talk to PostgreSQL.
+To save on round-trip latency, we batch our inserts using this code.
+
+In summary, committing 100 models in SQLAlchemy does 100 roundtrips
+to the database if the model has an autoincrementing primary key.
+This module improves this to 2 roundtrips without requiring any
+other changes to your code.
+
+## Usage
+
+```python
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy_batch_inserts import enable_batch_inserting
+
+engine = create_engine("postgresql+psycopg2://postgres@localhost", use_batch_mode=True)
+Session = sessionmaker(bind=engine)
+session = Session()
+enable_batch_inserting(session)
+```
+
+If you use [Flask-SQLALchemy](https://flask-sqlalchemy.palletsprojects.com/),
+
+```python
+from flask_sqlalchemy import SignallingSession
+from sqlalchemy_batch_inserts import enable_batch_inserting
+
+enable_batch_inserting(SignallingSession)
+```
+
+## Demo
+
+```
+docker build -t sqla_batch .
+docker run --rm -v $PWD:/src --name sqla_batch sqla_batch
+# Wait for it to finish spinning up
+# Switch to another shell
+docker exec -it sqla_batch src/demo.py no 100
+docker exec -it sqla_batch src/demo.py yes 100
+```
+
+To simulate 100 * 3 inserts with 20 ms latency,
+first change the connection string in demo.py from
+`postgresql+psycopg2://postgres@localhost` to `postgresql+psycopg2://postgres@db`.
+Then,
+```
+docker network create sqla_batch
+docker run --rm --network sqla_batch --network-alias db --name db sqla_batch
+# Switch to another shell
+docker run -it -v /var/run/docker.sock:/var/run/docker.sock --network sqla_batch gaiaadm/pumba netem --duration 15m --tc-image gaiadocker/iproute2 delay --time 20 --jitter 0 db
+# Switch to another shell
+# This should take 100 * 3 * 20 ms = 6 seconds
+docker run -it --rm -v $PWD:/src --network sqla_batch sqla_batch src/demo.py no 100
+docker run -it --rm -v $PWD:/src --network sqla_batch sqla_batch src/demo.py yes 100
+```
+
+## Maintainer notes
+
+After bumping the `version` in `setup.py` and `__version__` in `__init__.py`,
+
+```
+$ ./setup.py sdist bdist_wheel  # Generate source and py3 wheel
+$ python2 setup.py bdist_wheel  # Generate py2 wheel
+$ twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+# Check https://test.pypi.org/project/sqlalchemy-batch-inserts/
+$ twine upload dist/*
+```

demo.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+from contextlib import contextmanager
+import os
+import sys
+import time
+
+from sqlalchemy import create_engine, Column, ForeignKey, Integer, String
+from sqlalchemy.orm import relationship, sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+
+from sqlalchemy_batch_inserts import enable_batch_inserting
+
+Base = declarative_base()
+
+class User(Base):
+    __tablename__ = "users"
+
+    id = Column(Integer, primary_key=True)
+    name = Column(String, unique=True)
+
+    def __repr__(self):
+       return "<User(name=%r)>" % (self.name)
+
+class Address(Base):
+    __tablename__ = "addresses"
+    id = Column(Integer, primary_key=True)
+    email_address = Column(String, nullable=False, unique=True)
+    user_id = Column(Integer, ForeignKey("users.id"))
+
+    user = relationship("User")
+
+    def __repr__(self):
+        return "<Address(email_address=%r)>" % self.email_address
+
+
+def main():
+    if len(sys.argv) != 3:
+        usage_and_exit()
+    if sys.argv[1] == 'yes':
+        batch = True
+    elif sys.argv[1] == 'no':
+        batch = False
+    else:
+        usage_and_exit()
+    try:
+        count = int(sys.argv[2])
+    except ValueError:
+        usage_and_exit()
+
+    engine = create_engine("postgresql+psycopg2://postgres@localhost", use_batch_mode=True, echo=count <= 100)
+    Base.metadata.create_all(engine)
+
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    if batch:
+        enable_batch_inserting(session)
+
+    prefix = str(int(time.time()))
+    for i in range(count):
+        name = "%s-%d" % (prefix, i)
+        user = User(name=name)
+        user.addresses = [
+            Address(email_address="%[email protected]" % name),
+            Address(email_address="%[email protected]" % name),
+        ]
+        session.add(user)
+        session.add_all(user.addresses)
+
+    with Timer() as timer:
+        session.commit()
+    users = session.query(User).count()
+    addresses =session.query(Address).count()
+    print("took", timer.duration, "seconds")
+    print("have", users, "users")
+    print("have", addresses, "addresses")
+
+class Timer:
+    def __enter__(self):
+        self.start = time.time()
+        return self
+
+    def __exit__(self, exc_type, value, tb):
+        self.duration = time.time() - self.start
+
+def usage_and_exit():
+    sys.stderr.write(sys.argv[0] + " [batch] [count]\n")
+    sys.stderr.write(sys.argv[0] + " no 50\n")
+    sys.stderr.write(sys.argv[0] + " yes 1000\n")
+    sys.exit(1)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,2 @@
+psycopg2-binary
+sqlalchemy

setup.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import setuptools
+
+long_description = """
+Benchling uses SQLAlchemy and psycopg2 to talk to PostgreSQL.
+To save on round-trip latency, we batch our inserts using this code.
+
+## Usage
+
+```
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy_batch_inserts import enable_batch_inserting
+
+engine = create_engine("postgresql+psycopg2://postgres@localhost", use_batch_mode=True)
+Session = sessionmaker(bind=engine)
+session = Session()
+enable_batch_inserting(session)
+```
+
+If you use [Flask-SQLALchemy](https://flask-sqlalchemy.palletsprojects.com/),
+
+```
+from flask_sqlalchemy import SignallingSession
+
+enable_batch_inserting(SignallingSession)
+```
+"""
+
+setuptools.setup(
+    name="sqlalchemy_batch_inserts",
+    version="0.0.1",
+    author="Vineet Gopal",
+    author_email="[email protected]",
+    description="Batch inserts for SQLAlchemy on PostgreSQL with psycopg2",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/benchling/sqlalchemy_batch_inserts",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)