Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions ascender/conf/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,17 @@ def get_settings_to_cache(registry):
return dict([(key, SETTING_CACHE_NOTSET) for key in get_writeable_settings(registry)])


def _connect_outside_the_pool():
"""Give this thread its own connection rather than one from the pool.

The caller blocks the event loop until this thread answers, and requests
return their connections to the pool from that loop.
"""
options = connection.settings_dict.get('OPTIONS') or {}
if options.get('pool'):
connection.settings_dict = {**connection.settings_dict, 'OPTIONS': {k: v for k, v in options.items() if k != 'pool'}}


def _read_from_db(query):
"""Run a settings query, on another thread if this one runs an event loop.

Expand All @@ -249,6 +260,7 @@ def _read_from_db(query):
except SynchronousOnlyOperation:

def run():
_connect_outside_the_pool()
try:
return query()
finally:
Expand Down
36 changes: 36 additions & 0 deletions ascender/conf/tests/unit/test_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,5 +475,41 @@ def query():
return 'answered'

with mock.patch('ascender.conf.settings.connection') as conn:
conn.settings_dict = {}
assert _read_from_db(query) == 'answered'
assert conn.close.called, 'the thread left its connection open'


def test_the_thread_connects_outside_the_pool(monkeypatch):
"""Waiting on the pool would wait on connections that cannot come back."""
from django.db import connection, connections

from ascender.conf.settings import _read_from_db

shared = connections.settings['default']
monkeypatch.setitem(shared, 'OPTIONS', {**shared.get('OPTIONS', {}), 'pool': {'max_size': 1}, 'sslmode': 'prefer'})
seen = []

def query():
if not seen:
seen.append('first')
raise SynchronousOnlyOperation('cannot call this from an async context')
seen.append(dict(connection.settings_dict['OPTIONS']))
return 'answered'

assert _read_from_db(query) == 'answered'
assert 'pool' not in seen[-1], 'the thread would take its connection from the pool'
assert seen[-1]['sslmode'] == 'prefer', 'the thread lost its other connection options'
assert shared['OPTIONS']['pool'] == {'max_size': 1}, 'the pool was turned off for every thread'


def test_a_caller_that_stays_on_its_thread_keeps_the_pool(monkeypatch):
"""Only the thread hop leaves the pool."""
from django.db import connection, connections

from ascender.conf.settings import _read_from_db

shared = connections.settings['default']
monkeypatch.setitem(shared, 'OPTIONS', {**shared.get('OPTIONS', {}), 'pool': {'max_size': 1}})

assert _read_from_db(lambda: connection.settings_dict['OPTIONS'].get('pool')) == {'max_size': 1}
6 changes: 3 additions & 3 deletions ascender/main/tests/functional/test_licenses.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ def find_embedded_source_version(path, name):
files = os.listdir(path)
tgz_files = [f for f in files if f.endswith('.tar.gz')]
for tgz in tgz_files:
pkg_name = tgz.split('-')[0].split('_')[0]
if pkg_name == name:
return tgz.split('-')[1].split('.tar.gz')[0]
pkg_name, version = tgz[: -len('.tar.gz')].rsplit('-', 1)
if pkg_name.replace('_', '-').lower() == name:
return version
return None

list = {}
Expand Down
75 changes: 63 additions & 12 deletions ascender/main/tests/unit/settings/test_connection_reuse.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import pytest

from ascender.settings.connection_reuse import DEFAULT_WEB_CONN_MAX_AGE, is_web_process, set_conn_max_age
from ascender.settings.connection_reuse import DEFAULT_WEB_POOL, set_conn_max_age, set_connection_pool


@pytest.fixture
Expand All @@ -20,22 +20,13 @@ def no_web_marker(monkeypatch):
monkeypatch.delenv('AWX_WEB_PROCESS', raising=False)


class TestIsWebProcess:
def test_the_marker_is_a_web_process(self, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
assert is_web_process() is True

def test_anything_else_is_not(self):
assert is_web_process() is False


class TestWebProcess:
def test_a_web_process_keeps_its_connection(self, databases, monkeypatch):
def test_a_web_process_keeps_no_connection(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')

set_conn_max_age(databases)

assert databases['default']['CONN_MAX_AGE'] == DEFAULT_WEB_CONN_MAX_AGE
assert 'CONN_MAX_AGE' not in databases['default']

def test_the_setting_wins_over_the_default(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
Expand Down Expand Up @@ -78,3 +69,63 @@ def test_no_default_database(self):
set_conn_max_age(databases, 60)

assert databases == {'other': {}}


class TestPool:
def test_a_web_process_gets_the_pool(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')

set_connection_pool(databases)

assert databases['default']['OPTIONS']['pool'] == DEFAULT_WEB_POOL

def test_the_setting_replaces_the_sizes(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')

set_connection_pool(databases, {'min_size': 1, 'max_size': 10})

assert databases['default']['OPTIONS']['pool'] == {'min_size': 1, 'max_size': 10}

def test_the_setting_can_turn_it_off(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')

set_connection_pool(databases, False)

assert 'OPTIONS' not in databases['default']

def test_a_conn_max_age_leaves_it_off(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
set_conn_max_age(databases, 60)

set_connection_pool(databases)

assert 'OPTIONS' not in databases['default']

def test_a_conn_max_age_of_0_keeps_it(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
set_conn_max_age(databases, 0)

set_connection_pool(databases)

assert databases['default']['OPTIONS']['pool'] == DEFAULT_WEB_POOL

def test_other_options_are_kept(self, databases, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
databases['default']['OPTIONS'] = {'application_name': 'awx-web'}

set_connection_pool(databases)

assert databases['default']['OPTIONS'] == {'application_name': 'awx-web', 'pool': DEFAULT_WEB_POOL}

def test_a_task_process_gets_none(self, databases):
set_connection_pool(databases, {'max_size': 10})

assert 'OPTIONS' not in databases['default']

def test_no_default_database(self, monkeypatch):
monkeypatch.setenv('AWX_WEB_PROCESS', '1')
databases = {'other': {}}

set_connection_pool(databases)

assert databases == {'other': {}}
45 changes: 21 additions & 24 deletions ascender/settings/connection_reuse.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,14 @@
from ascender.settings.environment import environment_setting

# Django's default is 0: open a connection for each request and close it when
# the response is sent. For a process that serves one request after another
# that is pure churn, since the next request immediately needs another one,
# and a connection costs several milliseconds to establish before it has run
# any query at all.
#
# Sixty seconds is deliberately short. A worker that goes idle holds its
# connection until its next request notices the age, so the number held at
# rest is the number of web workers, and a short life keeps a connection from
# outliving a network path that has quietly gone away.
DEFAULT_WEB_CONN_MAX_AGE = 60


def is_web_process():
'''
Whether this process serves HTTP.

A web process says so with AWX_WEB_PROCESS, which the supervisor programs
that run uvicorn set. Nothing else here serves HTTP.
'''
return bool(environment_setting('WEB_PROCESS'))
DEFAULT_WEB_POOL = {'min_size': 2, 'max_size': 4}


def set_conn_max_age(DATABASES, DATABASE_CONN_MAX_AGE=None):
'''
Keep a web process's database connection alive between requests.
Leave every process at Django's default of 0 unless DATABASE_CONN_MAX_AGE
is set. A web process too: uvicorn runs each request in a thread of its
own, so a connection kept past the request is left behind with that thread
and never closed.

Task workers, management commands and migrations are left alone: they hold
one connection for the life of a long process, or want it closed the moment
Expand All @@ -40,7 +23,21 @@ def set_conn_max_age(DATABASES, DATABASE_CONN_MAX_AGE=None):

if DATABASE_CONN_MAX_AGE is not None:
DATABASES['default']['CONN_MAX_AGE'] = DATABASE_CONN_MAX_AGE


def set_connection_pool(DATABASES, DATABASE_POOL=None):
'''
Pool connections in a web process only: the dispatcher and wsrelay pass
OPTIONS straight to psycopg, which has no pool option.
'''
if not DATABASES or 'default' not in DATABASES:
return

if not environment_setting('WEB_PROCESS') or DATABASE_POOL is False:
return

if DATABASES['default'].get('CONN_MAX_AGE'):
return

if is_web_process():
DATABASES['default']['CONN_MAX_AGE'] = DEFAULT_WEB_CONN_MAX_AGE
options_dict = DATABASES['default'].setdefault('OPTIONS', dict())
options_dict['pool'] = dict(DEFAULT_WEB_POOL) if DATABASE_POOL is None else DATABASE_POOL
9 changes: 6 additions & 3 deletions ascender/settings/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,14 @@
JOB_EVENT_ARCHIVE_DIR = None

# Optional manual override for how long a database connection is kept and
# reused, in seconds. Unset, a web process reuses one for a minute and every
# other process opens one per request, which is Django's default. 0 turns
# reuse off everywhere.
# reused, in seconds. Unset or 0, a web process takes its connections from the
# pool below and every other process opens one per request, which is Django's
# default. Above 0, a web process does not pool.
DATABASE_CONN_MAX_AGE = None

# Pool options for web processes, passed to psycopg_pool. False turns it off.
DATABASE_POOL = None

# Special database overrides for dispatcher connections listening to pg_notify
LISTENER_DATABASES = {
'default': {
Expand Down
5 changes: 3 additions & 2 deletions ascender/settings/development.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,9 @@

del set_statement_timeout

from .connection_reuse import set_conn_max_age
from .connection_reuse import set_conn_max_age, set_connection_pool

set_conn_max_age(DATABASES, DATABASE_CONN_MAX_AGE) # NOQA
set_connection_pool(DATABASES, DATABASE_POOL) # NOQA

del set_conn_max_age
del set_conn_max_age, set_connection_pool
5 changes: 3 additions & 2 deletions ascender/settings/production.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,12 @@

del set_statement_timeout

from .connection_reuse import set_conn_max_age
from .connection_reuse import set_conn_max_age, set_connection_pool

set_conn_max_age(DATABASES, DATABASE_CONN_MAX_AGE) # NOQA
set_connection_pool(DATABASES, DATABASE_POOL) # NOQA

del set_conn_max_age
del set_conn_max_age, set_connection_pool

# Deployments that set a former name in /etc/tower/conf.d keep working for a
# release. The stored values are carried over by conf migrations 0011 and 0012,
Expand Down
1 change: 1 addition & 0 deletions ascender/settings/typed.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ class AscenderSettings(Protocol):
CUSTOM_TITLE: str
DATABASES: dict[str, Any]
DATABASE_CONN_MAX_AGE: Any
DATABASE_POOL: Any
DATABASE_ROUTERS: list[Any]
DATABASE_STATEMENT_TIMEOUT: Any
DATA_UPLOAD_MAX_MEMORY_SIZE: int
Expand Down
Loading