Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions ascender/api/serializers/inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ class Meta:
'prevent_instance_group_fallback',
'allow_deletes_while_in_use',
'allow_jobs_while_syncing',
'variables_from_sources_only',
)
extra_kwargs = {
# required/default must be explicit since DRF 3.16: nullable FKs now
Expand Down Expand Up @@ -140,6 +141,8 @@ def validate(self, attrs):

if kind == 'smart' and not host_filter:
raise serializers.ValidationError({'host_filter': _('Smart inventories must specify host_filter')})
if kind and attrs.get('variables_from_sources_only'):
raise serializers.ValidationError({'variables_from_sources_only': _('Only regular inventories have inventory sources of their own.')})
return super(InventorySerializer, self).validate(attrs)


Expand Down Expand Up @@ -190,7 +193,7 @@ class ConstructedInventorySerializer(InventorySerializer):

class Meta:
model = Inventory
fields = ('*', '-host_filter') + CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS
fields = ('*', '-host_filter', '-variables_from_sources_only') + CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS
read_only_fields = ('*', 'kind')

def pop_inv_src_data(self, data):
Expand Down Expand Up @@ -231,7 +234,7 @@ def update(self, obj, validated_data):
class FederatedInventorySerializer(InventorySerializer):
class Meta:
model = Inventory
fields = ('*', '-host_filter', '-instance_groups', '-prevent_instance_group_fallback')
fields = ('*', '-host_filter', '-instance_groups', '-prevent_instance_group_fallback', '-variables_from_sources_only')
read_only_fields = ('*', 'kind')

def create(self, validated_data):
Expand Down
35 changes: 31 additions & 4 deletions ascender/main/management/commands/inventory_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,17 +466,41 @@ def _remove_stale_inventory_variables(self, db_variables, source_variables, prev
del db_variables[key]
logger.debug('Inventory variable "%s" removed, no longer provided by the source', key)

def _remove_variables_not_from_sources(self, db_variables, source_variables):
"""
Remove inventory variables that no source of this inventory provides. Skipped while
any other source has not recorded its variables yet, since its keys are unknown.
"""
other_sources = self.inventory.inventory_sources.exclude(pk=self.inventory_source.pk)
unrecorded = list(other_sources.filter(managed_inventory_variables_recorded=False).values_list('name', flat=True))
if unrecorded:
logger.warning(
'Inventory variables not provided by any source were kept: %s must sync before they can be removed',
', '.join('"{}"'.format(name) for name in sorted(unrecorded)),
)
return
provided = set(source_variables)
for other_managed in other_sources.values_list('managed_inventory_variables', flat=True):
provided.update(other_managed)
for key in [key for key in db_variables if key not in provided]:
del db_variables[key]
logger.info('Inventory variable "%s" removed, no source of this inventory provides it', key)

def _update_inventory(self):
"""
Update inventory variables from "all" group.
"""
# Merged rather than replaced so variables from users and other sources survive; with
# overwrite_vars, only variables this source previously wrote are removed. See ansible/awx#11623.
# With variables_from_sources_only, variables no source of the inventory provides are removed too.
source_variables = self.all_group.variables
# Reread under the perform_update lock; the cached inventory and source may predate a concurrent sync.
self.inventory.refresh_from_db(fields=['variables'])
self.inventory.refresh_from_db(fields=['variables', 'variables_from_sources_only'])
original_variables = self.inventory.variables_dict
previous = InventorySource.objects.filter(pk=self.inventory_source.pk).values_list('managed_inventory_variables', flat=True).first() or {}
previous, recorded = InventorySource.objects.filter(pk=self.inventory_source.pk).values_list(
'managed_inventory_variables', 'managed_inventory_variables_recorded'
).first() or ({}, False)
previous = previous or {}

if self.inventory.kind == 'constructed' and self.inventory_source.overwrite_vars:
# NOTE: we had to add a exception case to not merge variables
Expand All @@ -486,6 +510,8 @@ def _update_inventory(self):
db_variables = dict(original_variables)
if self.overwrite_vars:
self._remove_stale_inventory_variables(db_variables, source_variables, previous)
if self.inventory.variables_from_sources_only:
self._remove_variables_not_from_sources(db_variables, source_variables)
db_variables.update(source_variables)

if db_variables != original_variables:
Expand All @@ -496,10 +522,11 @@ def _update_inventory(self):
logger.debug('Inventory variables unmodified')

managed = {key: self._hash_variable(value) for key, value in source_variables.items()}
if managed != previous:
if managed != previous or not recorded:
# update() skips InventorySource.save(), which recomputes inventory fields and fires signals.
InventorySource.objects.filter(pk=self.inventory_source.pk).update(managed_inventory_variables=managed)
InventorySource.objects.filter(pk=self.inventory_source.pk).update(managed_inventory_variables=managed, managed_inventory_variables_recorded=True)
self.inventory_source.managed_inventory_variables = managed
self.inventory_source.managed_inventory_variables_recorded = True

def _create_update_groups(self):
"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
('main', '0233_job_monitor_lease'),
]

operations = [
migrations.AddField(
model_name='inventory',
name='variables_from_sources_only',
field=models.BooleanField(
default=False,
help_text='If enabled, each sync of an inventory source removes the inventory variables that no source of this inventory provides, including variables added by hand or left over from older syncs. Nothing is removed until every source of the inventory has synced at least once.',
),
),
migrations.AddField(
model_name='inventorysource',
name='managed_inventory_variables_recorded',
field=models.BooleanField(
default=False,
editable=False,
help_text='Whether a sync of this source has recorded managed_inventory_variables.',
),
),
]
14 changes: 14 additions & 0 deletions ascender/main/models/inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
'prevent_instance_group_fallback',
'allow_deletes_while_in_use',
'allow_jobs_while_syncing',
'variables_from_sources_only',
]
KIND_CHOICES = [
('', _('Hosts have a direct link to this inventory.')),
Expand Down Expand Up @@ -233,6 +234,14 @@ class Meta:
'and for a running one when that source has never synced successfully.'
),
)
variables_from_sources_only = models.BooleanField(
default=False,
help_text=_(
'If enabled, each sync of an inventory source removes the inventory variables that no source '
'of this inventory provides, including variables added by hand or left over from older syncs. '
'Nothing is removed until every source of the inventory has synced at least once.'
),
)

def get_absolute_url(self, request=None):
if request is not None:
Expand Down Expand Up @@ -1179,6 +1188,11 @@ class Meta:
editable=False,
help_text=_('Inventory variable names written by the last sync of this source, mapped to a hash of the value written.'),
)
managed_inventory_variables_recorded = models.BooleanField(
default=False,
editable=False,
help_text=_('Whether a sync of this source has recorded managed_inventory_variables.'),
)

@classmethod
def _get_unified_job_class(cls):
Expand Down
32 changes: 32 additions & 0 deletions ascender/main/tests/functional/api/test_inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,38 @@ def test_allow_jobs_while_syncing_is_editable(get, patch, organization, admin, k
assert inventory.allow_jobs_while_syncing is True


@pytest.mark.django_db
def test_variables_from_sources_only_is_editable(get, patch, inventory, admin):
url = reverse('api:inventory_detail', kwargs={'pk': inventory.pk})
assert get(url, admin, expect=200).data['variables_from_sources_only'] is False

patch(url, {'variables_from_sources_only': True}, admin, expect=200)
inventory.refresh_from_db()
assert inventory.variables_from_sources_only is True


@pytest.mark.django_db
def test_variables_from_sources_only_rejected_on_smart_inventory(post, patch, organization, admin):
data = {'name': 'smart', 'kind': 'smart', 'host_filter': 'name=foo', 'organization': organization.pk, 'variables_from_sources_only': True}
resp = post(reverse('api:inventory_list'), data, admin, expect=400)
assert 'variables_from_sources_only' in resp.data

smart = Inventory.objects.create(name='smart2', kind='smart', host_filter='name=foo', organization=organization)
patch(reverse('api:inventory_detail', kwargs={'pk': smart.pk}), {'variables_from_sources_only': True}, admin, expect=400)
# Turning it off is always allowed.
patch(reverse('api:inventory_detail', kwargs={'pk': smart.pk}), {'variables_from_sources_only': False}, admin, expect=200)


@pytest.mark.django_db
def test_variables_from_sources_only_not_on_constructed_inventory(get, patch, organization, admin):
inventory = Inventory.objects.create(name='inv', kind='constructed', organization=organization)
url = reverse('api:constructed_inventory_detail', kwargs={'pk': inventory.pk})
assert 'variables_from_sources_only' not in get(url, admin, expect=200).data
patch(url, {'variables_from_sources_only': True}, admin, expect=200)
inventory.refresh_from_db()
assert inventory.variables_from_sources_only is False


@pytest.mark.parametrize('order_by', ('extra_vars', '-extra_vars', 'extra_vars,pk', '-extra_vars,pk'))
@pytest.mark.django_db
def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by):
Expand Down
99 changes: 99 additions & 0 deletions ascender/main/tests/functional/commands/test_inventory_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,105 @@ def test_failed_sync_does_not_update_tracking(self, inventory):
assert self.sync(inv_src, {'a': 1}) == {'a': 1}


@pytest.mark.django_db
@mock.patch.object(inventory_import.Command, 'set_logging_level', mock_logging)
class TestInventoryVariablesFromSourcesOnly:
@pytest.fixture
def inventory(self, inventory):
inventory.variables_from_sources_only = True
inventory.save()
return inventory

def sync(self, inv_src, variables, overwrite_vars=True):
inventory_import.Command().perform_update(dict(overwrite_vars=overwrite_vars), all_vars_data(variables), inv_src.create_unified_job())
inv_src.inventory.refresh_from_db()
return inv_src.inventory.variables_dict

def test_untracked_variables_removed(self, inventory):
# Keys left by syncs from before the source tracked its variables, or added by hand.
inventory.variables = '{"legacy": 1, "mine": "x"}'
inventory.save()
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2', overwrite_vars=True)
assert self.sync(inv_src, {'a': 1}) == {'a': 1}

def test_untracked_variables_kept_when_disabled(self, inventory):
inventory.variables = '{"legacy": 1}'
inventory.variables_from_sources_only = False
inventory.save()
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2', overwrite_vars=True)
assert self.sync(inv_src, {'a': 1}) == {'legacy': 1, 'a': 1}

def test_removed_variable_deleted_even_if_edited(self, inventory):
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2', overwrite_vars=True)
self.sync(inv_src, {'a': 1, 'b': 2})
inventory.variables = '{"a": 1, "b": "edited"}'
inventory.save()
assert self.sync(inv_src, {'a': 1}) == {'a': 1}

def test_edited_variable_still_provided_is_overwritten(self, inventory):
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2', overwrite_vars=True)
self.sync(inv_src, {'a': 1})
inventory.variables = '{"a": "edited"}'
inventory.save()
assert self.sync(inv_src, {'a': 1}) == {'a': 1}

def test_applies_without_overwrite_vars(self, inventory):
inventory.variables = '{"legacy": 1}'
inventory.save()
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2')
assert self.sync(inv_src, {'a': 1, 'b': 2}, overwrite_vars=False) == {'a': 1, 'b': 2}
assert self.sync(inv_src, {'a': 1}, overwrite_vars=False) == {'a': 1}

def test_variables_of_other_sources_are_kept(self, inventory):
src_a = InventorySource.objects.create(inventory=inventory, name='a', source='ec2', overwrite_vars=True)
src_b = InventorySource.objects.create(inventory=inventory, name='b', source='ec2', overwrite_vars=True)
self.sync(src_a, {'only_a': 1, 'shared': 1})
self.sync(src_b, {'only_b': 2, 'shared': 2})
inventory.variables = '{"only_a": 1, "only_b": 2, "shared": 2, "mine": "x"}'
inventory.save()
assert self.sync(src_a, {'shared': 1}) == {'only_b': 2, 'shared': 1}

def test_nothing_removed_until_every_source_has_synced(self, inventory):
inventory.variables = '{"legacy": 1}'
inventory.save()
src_a = InventorySource.objects.create(inventory=inventory, name='a', source='ec2', overwrite_vars=True)
src_b = InventorySource.objects.create(inventory=inventory, name='b', source='ec2', overwrite_vars=True)
# b has not recorded its variables yet, so "legacy" could be one of them.
assert self.sync(src_a, {'a': 1}) == {'legacy': 1, 'a': 1}
assert self.sync(src_b, {'b': 2}) == {'a': 1, 'b': 2}

def test_source_without_variables_counts_as_synced(self, inventory):
src_a = InventorySource.objects.create(inventory=inventory, name='a', source='ec2', overwrite_vars=True)
src_b = InventorySource.objects.create(inventory=inventory, name='b', source='ec2', overwrite_vars=True)
self.sync(src_b, {})
src_b.refresh_from_db()
assert src_b.managed_inventory_variables == {}
assert src_b.managed_inventory_variables_recorded is True
inventory.variables = '{"legacy": 1}'
inventory.save()
assert self.sync(src_a, {'a': 1}) == {'a': 1}

def test_variables_of_deleted_source_removed_on_next_sync(self, inventory):
src_a = InventorySource.objects.create(inventory=inventory, name='a', source='ec2', overwrite_vars=True)
src_b = InventorySource.objects.create(inventory=inventory, name='b', source='ec2', overwrite_vars=True)
self.sync(src_a, {'a': 1})
self.sync(src_b, {'b': 2})
src_b.delete()
assert self.sync(src_a, {'a': 1}) == {'a': 1}

def test_failed_sync_keeps_variables(self, inventory):
inventory.variables = '{"legacy": 1}'
inventory.save()
inv_src = InventorySource.objects.create(inventory=inventory, source='ec2', overwrite_vars=True)
with mock.patch.object(inventory_import.Command, '_create_update_groups', side_effect=RuntimeError):
with pytest.raises(RuntimeError):
self.sync(inv_src, {'a': 1})
inventory.refresh_from_db()
assert inventory.variables_dict == {'legacy': 1}
inv_src.refresh_from_db()
assert inv_src.managed_inventory_variables_recorded is False


@pytest.mark.django_db
@pytest.mark.inventory_import
class TestEnabledVar:
Expand Down
2 changes: 2 additions & 0 deletions ascender/main/tests/functional/test_copy.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def test_inventory_copy(inventory, group_factory, post, get, alice, organization
inventory.admin_role.members.add(alice)
inventory.prevent_instance_group_fallback = True
inventory.allow_jobs_while_syncing = True
inventory.variables_from_sources_only = True
inventory.save()
assert get(reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200).data['can_copy'] is False
inventory.organization.admin_role.members.add(alice)
Expand All @@ -117,6 +118,7 @@ def test_inventory_copy(inventory, group_factory, post, get, alice, organization
assert inventory_copy.name == 'new inv name'
assert inventory_copy.prevent_instance_group_fallback == True
assert inventory_copy.allow_jobs_while_syncing is True
assert inventory_copy.variables_from_sources_only is True
assert set(group_1_1_copy.parents.all()) == set()
assert set(group_2_1_copy.parents.all()) == set([group_1_1_copy])
assert set(group_2_2_copy.parents.all()) == set([group_1_1_copy, group_2_1_copy])
Expand Down
2 changes: 1 addition & 1 deletion ascender/ui/src/locales/ar/messages.mjs

Large diffs are not rendered by default.

Loading