1136 lines
55 KiB
Python
1136 lines
55 KiB
Python
# Copyright (c) 2015 Ansible, Inc.
|
|
# All Rights Reserved.
|
|
|
|
# Python
|
|
import json
|
|
import logging
|
|
import fnmatch
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import shutil
|
|
|
|
# Django
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand, CommandError
|
|
from django.db import connection, transaction
|
|
from django.utils.encoding import smart_text
|
|
|
|
# DRF error class to distinguish license exceptions
|
|
from rest_framework.exceptions import PermissionDenied
|
|
|
|
# AWX inventory imports
|
|
from awx.main.models.inventory import (
|
|
Inventory,
|
|
InventorySource,
|
|
InventoryUpdate,
|
|
Host
|
|
)
|
|
from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data
|
|
from awx.main.utils.safe_yaml import sanitize_jinja
|
|
|
|
# other AWX imports
|
|
from awx.main.models.rbac import batch_role_ancestor_rebuilding
|
|
# TODO: remove proot utils once we move to running inv. updates in containers
|
|
from awx.main.utils import (
|
|
check_proot_installed,
|
|
wrap_args_with_proot,
|
|
build_proot_temp_dir,
|
|
ignore_inventory_computed_fields,
|
|
get_licenser
|
|
)
|
|
from awx.main.signals import disable_activity_stream
|
|
from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
|
|
from awx.main.utils.pglock import advisory_lock
|
|
|
|
logger = logging.getLogger('awx.main.commands.inventory_import')
|
|
|
|
LICENSE_EXPIRED_MESSAGE = '''\
|
|
License expired.
|
|
See http://www.ansible.com/renew for license extension information.'''
|
|
|
|
LICENSE_NON_EXISTANT_MESSAGE = '''\
|
|
No license.
|
|
See http://www.ansible.com/renew for license information.'''
|
|
|
|
LICENSE_MESSAGE = '''\
|
|
Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(instance_count)d.
|
|
See http://www.ansible.com/renew for license extension information.'''
|
|
|
|
DEMO_LICENSE_MESSAGE = '''\
|
|
Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(instance_count)d.
|
|
See http://www.ansible.com/renew for licensing information.'''
|
|
|
|
|
|
def functioning_dir(path):
|
|
if os.path.isdir(path):
|
|
return path
|
|
return os.path.dirname(path)
|
|
|
|
|
|
class AnsibleInventoryLoader(object):
|
|
'''
|
|
Given executable `source` (directory, executable, or file) this will
|
|
use the ansible-inventory CLI utility to convert it into in-memory
|
|
representational objects. Example:
|
|
/usr/bin/ansible/ansible-inventory -i hosts --list
|
|
'''
|
|
|
|
def __init__(self, source, venv_path=None, verbosity=0):
|
|
self.source = source
|
|
self.verbosity = verbosity
|
|
# TODO: remove once proot has been removed
|
|
self.tmp_private_dir = None
|
|
if venv_path:
|
|
self.venv_path = venv_path
|
|
else:
|
|
self.venv_path = settings.ANSIBLE_VENV_PATH
|
|
|
|
def build_env(self):
|
|
env = dict(os.environ.items())
|
|
env['VIRTUAL_ENV'] = self.venv_path
|
|
env['PATH'] = os.path.join(self.venv_path, "bin") + ":" + env['PATH']
|
|
# Set configuration items that should always be used for updates
|
|
for key, value in STANDARD_INVENTORY_UPDATE_ENV.items():
|
|
if key not in env:
|
|
env[key] = value
|
|
venv_libdir = os.path.join(self.venv_path, "lib")
|
|
env.pop('PYTHONPATH', None) # default to none if no python_ver matches
|
|
for version in os.listdir(venv_libdir):
|
|
if fnmatch.fnmatch(version, 'python[23].*'):
|
|
if os.path.isdir(os.path.join(venv_libdir, version)):
|
|
env['PYTHONPATH'] = os.path.join(venv_libdir, version, "site-packages") + ":"
|
|
break
|
|
# For internal inventory updates, these are not reported in the job_env API
|
|
logger.info('Using VIRTUAL_ENV: {}'.format(env['VIRTUAL_ENV']))
|
|
logger.info('Using PATH: {}'.format(env['PATH']))
|
|
logger.info('Using PYTHONPATH: {}'.format(env.get('PYTHONPATH', None)))
|
|
return env
|
|
|
|
def get_path_to_ansible_inventory(self):
|
|
venv_exe = os.path.join(self.venv_path, 'bin', 'ansible-inventory')
|
|
if os.path.exists(venv_exe):
|
|
return venv_exe
|
|
elif os.path.exists(
|
|
os.path.join(self.venv_path, 'bin', 'ansible')
|
|
):
|
|
# if bin/ansible exists but bin/ansible-inventory doesn't, it's
|
|
# probably a really old version of ansible that doesn't support
|
|
# ansible-inventory
|
|
raise RuntimeError(
|
|
"{} does not exist (please upgrade to ansible >= 2.4)".format(
|
|
venv_exe
|
|
)
|
|
)
|
|
return shutil.which('ansible-inventory')
|
|
|
|
def get_base_args(self):
|
|
# get ansible-inventory absolute path for running in bubblewrap/proot, in Popen
|
|
ansible_inventory_path = self.get_path_to_ansible_inventory()
|
|
# NOTE: why do we add "python" to the start of these args?
|
|
# the script that runs ansible-inventory specifies a python interpreter
|
|
# that makes no sense in light of the fact that we put all the dependencies
|
|
# inside of /var/lib/awx/venv/ansible, so we override the specified interpreter
|
|
# https://github.com/ansible/ansible/issues/50714
|
|
bargs = ['python', ansible_inventory_path, '-i', self.source]
|
|
bargs.extend(['--playbook-dir', functioning_dir(self.source)])
|
|
if self.verbosity:
|
|
# INFO: -vvv, DEBUG: -vvvvv, for inventory, any more than 3 makes little difference
|
|
bargs.append('-{}'.format('v' * min(5, self.verbosity * 2 + 1)))
|
|
logger.debug('Using base command: {}'.format(' '.join(bargs)))
|
|
return bargs
|
|
|
|
# TODO: Remove this once we move to running ansible-inventory in containers
|
|
# and don't need proot for process isolation anymore
|
|
def get_proot_args(self, cmd, env):
|
|
cwd = os.getcwd()
|
|
if not check_proot_installed():
|
|
raise RuntimeError("proot is not installed but is configured for use")
|
|
|
|
kwargs = {}
|
|
# we cannot safely store tmp data in source dir or trust script contents
|
|
if env['AWX_PRIVATE_DATA_DIR']:
|
|
# If this is non-blank, file credentials are being used and we need access
|
|
private_data_dir = functioning_dir(env['AWX_PRIVATE_DATA_DIR'])
|
|
logger.debug("Using private credential data in '{}'.".format(private_data_dir))
|
|
kwargs['private_data_dir'] = private_data_dir
|
|
self.tmp_private_dir = build_proot_temp_dir()
|
|
logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir))
|
|
kwargs['proot_temp_dir'] = self.tmp_private_dir
|
|
kwargs['proot_show_paths'] = [functioning_dir(self.source), settings.AWX_ANSIBLE_COLLECTIONS_PATHS]
|
|
logger.debug("Running from `{}` working directory.".format(cwd))
|
|
|
|
if self.venv_path != settings.ANSIBLE_VENV_PATH:
|
|
kwargs['proot_custom_virtualenv'] = self.venv_path
|
|
|
|
return wrap_args_with_proot(cmd, cwd, **kwargs)
|
|
|
|
|
|
def command_to_json(self, cmd):
|
|
data = {}
|
|
stdout, stderr = '', ''
|
|
env = self.build_env()
|
|
|
|
# TODO: remove proot args once inv. updates run in containers
|
|
if (('AWX_PRIVATE_DATA_DIR' in env) and
|
|
getattr(settings, 'AWX_PROOT_ENABLED', False)):
|
|
cmd = self.get_proot_args(cmd, env)
|
|
|
|
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
|
|
stdout, stderr = proc.communicate()
|
|
stdout = smart_text(stdout)
|
|
stderr = smart_text(stderr)
|
|
|
|
# TODO: can be removed when proot is removed
|
|
if self.tmp_private_dir:
|
|
shutil.rmtree(self.tmp_private_dir, True)
|
|
|
|
if proc.returncode != 0:
|
|
raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % (
|
|
'ansible-inventory', proc.returncode, stdout, stderr))
|
|
|
|
for line in stderr.splitlines():
|
|
logger.error(line)
|
|
try:
|
|
data = json.loads(stdout)
|
|
if not isinstance(data, dict):
|
|
raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
|
|
except Exception:
|
|
logger.error('Failed to load JSON from: %s', stdout)
|
|
raise
|
|
return data
|
|
|
|
def load(self):
|
|
base_args = self.get_base_args()
|
|
logger.info('Reading Ansible inventory source: %s', self.source)
|
|
|
|
return self.command_to_json(base_args + ['--list'])
|
|
|
|
|
|
class Command(BaseCommand):
|
|
'''
|
|
Management command to import inventory from a directory, ini file, or
|
|
dynamic inventory script.
|
|
'''
|
|
|
|
help = 'Import or sync external inventory sources'
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('--inventory-name', dest='inventory_name',
|
|
type=str, default=None, metavar='n',
|
|
help='name of inventory to sync')
|
|
parser.add_argument('--inventory-id', dest='inventory_id', type=int,
|
|
default=None, metavar='i',
|
|
help='id of inventory to sync')
|
|
parser.add_argument('--venv', dest='venv', type=str, default=None,
|
|
help='absolute path to the AWX custom virtualenv to use')
|
|
parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False,
|
|
help='overwrite the destination hosts and groups')
|
|
parser.add_argument('--overwrite-vars', dest='overwrite_vars',
|
|
action='store_true', default=False,
|
|
help='overwrite (rather than merge) variables')
|
|
parser.add_argument('--keep-vars', dest='keep_vars', action='store_true', default=False,
|
|
help='DEPRECATED legacy option, has no effect')
|
|
parser.add_argument('--custom', dest='custom', action='store_true', default=False,
|
|
help='DEPRECATED indicates a custom inventory script, no longer used')
|
|
parser.add_argument('--source', dest='source', type=str, default=None,
|
|
metavar='s', help='inventory directory, file, or script to load')
|
|
parser.add_argument('--enabled-var', dest='enabled_var', type=str,
|
|
default=None, metavar='v', help='host variable used to '
|
|
'set/clear enabled flag when host is online/offline, may '
|
|
'be specified as "foo.bar" to traverse nested dicts.')
|
|
parser.add_argument('--enabled-value', dest='enabled_value', type=str,
|
|
default=None, metavar='v', help='value of host variable '
|
|
'specified by --enabled-var that indicates host is '
|
|
'enabled/online.')
|
|
parser.add_argument('--group-filter', dest='group_filter', type=str,
|
|
default=None, metavar='regex', help='regular expression '
|
|
'to filter group name(s); only matches are imported.')
|
|
parser.add_argument('--host-filter', dest='host_filter', type=str,
|
|
default=None, metavar='regex', help='regular expression '
|
|
'to filter host name(s); only matches are imported.')
|
|
parser.add_argument('--exclude-empty-groups', dest='exclude_empty_groups',
|
|
action='store_true', default=False, help='when set, '
|
|
'exclude all groups that have no child groups, hosts, or '
|
|
'variables.')
|
|
parser.add_argument('--instance-id-var', dest='instance_id_var', type=str,
|
|
default=None, metavar='v', help='host variable that '
|
|
'specifies the unique, immutable instance ID, may be '
|
|
'specified as "foo.bar" to traverse nested dicts.')
|
|
|
|
def set_logging_level(self, verbosity):
|
|
log_levels = dict(enumerate([logging.WARNING, logging.INFO,
|
|
logging.DEBUG, 0]))
|
|
logger.setLevel(log_levels.get(verbosity, 0))
|
|
|
|
def _get_instance_id(self, variables, default=''):
|
|
'''
|
|
Retrieve the instance ID from the given dict of host variables.
|
|
|
|
The instance ID variable may be specified as 'foo.bar', in which case
|
|
the lookup will traverse into nested dicts, equivalent to:
|
|
|
|
from_dict.get('foo', {}).get('bar', default)
|
|
|
|
Multiple ID variables may be specified as 'foo.bar,foobar', so that
|
|
it will first try to find 'bar' inside of 'foo', and if unable,
|
|
will try to find 'foobar' as a fallback
|
|
'''
|
|
instance_id = default
|
|
if getattr(self, 'instance_id_var', None):
|
|
for single_instance_id in self.instance_id_var.split(','):
|
|
from_dict = variables
|
|
for key in single_instance_id.split('.'):
|
|
if not hasattr(from_dict, 'get'):
|
|
instance_id = default
|
|
break
|
|
instance_id = from_dict.get(key, default)
|
|
from_dict = instance_id
|
|
if instance_id:
|
|
break
|
|
return smart_text(instance_id)
|
|
|
|
def _get_enabled(self, from_dict, default=None):
|
|
'''
|
|
Retrieve the enabled state from the given dict of host variables.
|
|
|
|
The enabled variable may be specified as 'foo.bar', in which case
|
|
the lookup will traverse into nested dicts, equivalent to:
|
|
|
|
from_dict.get('foo', {}).get('bar', default)
|
|
'''
|
|
enabled = default
|
|
if getattr(self, 'enabled_var', None):
|
|
default = object()
|
|
for key in self.enabled_var.split('.'):
|
|
if not hasattr(from_dict, 'get'):
|
|
enabled = default
|
|
break
|
|
enabled = from_dict.get(key, default)
|
|
from_dict = enabled
|
|
if enabled is not default:
|
|
enabled_value = getattr(self, 'enabled_value', None)
|
|
if enabled_value is not None:
|
|
enabled = bool(str(enabled_value).lower() == str(enabled).lower())
|
|
else:
|
|
enabled = bool(enabled)
|
|
if enabled is default:
|
|
return None
|
|
elif isinstance(enabled, bool):
|
|
return enabled
|
|
else:
|
|
raise NotImplementedError('Value of enabled {} not understood.'.format(enabled))
|
|
|
|
@staticmethod
|
|
def get_source_absolute_path(source):
|
|
if not os.path.exists(source):
|
|
raise IOError('Source does not exist: %s' % source)
|
|
source = os.path.join(os.getcwd(), os.path.dirname(source),
|
|
os.path.basename(source))
|
|
source = os.path.normpath(os.path.abspath(source))
|
|
return source
|
|
|
|
def _batch_add_m2m(self, related_manager, *objs, **kwargs):
|
|
key = (related_manager.instance.pk, related_manager.through._meta.db_table)
|
|
flush = bool(kwargs.get('flush', False))
|
|
if not hasattr(self, '_batch_add_m2m_cache'):
|
|
self._batch_add_m2m_cache = {}
|
|
cached_objs = self._batch_add_m2m_cache.setdefault(key, [])
|
|
cached_objs.extend(objs)
|
|
if len(cached_objs) > self._batch_size or flush:
|
|
if len(cached_objs):
|
|
related_manager.add(*cached_objs)
|
|
self._batch_add_m2m_cache[key] = []
|
|
|
|
def _build_db_instance_id_map(self):
|
|
'''
|
|
Find any hosts in the database without an instance_id set that may
|
|
still have one available via host variables.
|
|
'''
|
|
self.db_instance_id_map = {}
|
|
if self.instance_id_var:
|
|
host_qs = self.inventory_source.hosts.all()
|
|
host_qs = host_qs.filter(instance_id='',
|
|
variables__contains=self.instance_id_var.split('.')[0])
|
|
for host in host_qs:
|
|
instance_id = self._get_instance_id(host.variables_dict)
|
|
if not instance_id:
|
|
continue
|
|
self.db_instance_id_map[instance_id] = host.pk
|
|
|
|
def _build_mem_instance_id_map(self):
|
|
'''
|
|
Update instance ID for each imported host and define a mapping of
|
|
instance IDs to MemHost instances.
|
|
'''
|
|
self.mem_instance_id_map = {}
|
|
if self.instance_id_var:
|
|
for mem_host in self.all_group.all_hosts.values():
|
|
instance_id = self._get_instance_id(mem_host.variables)
|
|
if not instance_id:
|
|
logger.warning('Host "%s" has no "%s" variable(s)',
|
|
mem_host.name, self.instance_id_var)
|
|
continue
|
|
mem_host.instance_id = instance_id
|
|
self.mem_instance_id_map[instance_id] = mem_host.name
|
|
|
|
def _existing_host_pks(self):
|
|
'''Returns cached set of existing / previous host primary key values
|
|
this is the starting set, meaning that it is pre-modification
|
|
by deletions and other things done in the course of this import
|
|
'''
|
|
if not hasattr(self, '_cached_host_pk_set'):
|
|
self._cached_host_pk_set = frozenset(
|
|
self.inventory_source.hosts.values_list('pk', flat=True))
|
|
return self._cached_host_pk_set
|
|
|
|
def _delete_hosts(self):
|
|
'''
|
|
For each host in the database that is NOT in the local list, delete
|
|
it. When importing from a cloud inventory source attached to a
|
|
specific group, only delete hosts beneath that group. Delete each
|
|
host individually so signal handlers will run.
|
|
'''
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
hosts_qs = self.inventory_source.hosts
|
|
# Build list of all host pks, remove all that should not be deleted.
|
|
del_host_pks = set(self._existing_host_pks()) # makes mutable copy
|
|
if self.instance_id_var:
|
|
all_instance_ids = list(self.mem_instance_id_map.keys())
|
|
instance_ids = []
|
|
for offset in range(0, len(all_instance_ids), self._batch_size):
|
|
instance_ids = all_instance_ids[offset:(offset + self._batch_size)]
|
|
for host_pk in hosts_qs.filter(instance_id__in=instance_ids).values_list('pk', flat=True):
|
|
del_host_pks.discard(host_pk)
|
|
for host_pk in set([v for k,v in self.db_instance_id_map.items() if k in instance_ids]):
|
|
del_host_pks.discard(host_pk)
|
|
all_host_names = list(set(self.mem_instance_id_map.values()) - set(self.all_group.all_hosts.keys()))
|
|
else:
|
|
all_host_names = list(self.all_group.all_hosts.keys())
|
|
for offset in range(0, len(all_host_names), self._batch_size):
|
|
host_names = all_host_names[offset:(offset + self._batch_size)]
|
|
for host_pk in hosts_qs.filter(name__in=host_names).values_list('pk', flat=True):
|
|
del_host_pks.discard(host_pk)
|
|
# Now delete all remaining hosts in batches.
|
|
all_del_pks = sorted(list(del_host_pks))
|
|
for offset in range(0, len(all_del_pks), self._batch_size):
|
|
del_pks = all_del_pks[offset:(offset + self._batch_size)]
|
|
for host in hosts_qs.filter(pk__in=del_pks):
|
|
host_name = host.name
|
|
host.delete()
|
|
logger.debug('Deleted host "%s"', host_name)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('host deletions took %d queries for %d hosts',
|
|
len(connection.queries) - queries_before,
|
|
len(all_del_pks))
|
|
|
|
def _delete_groups(self):
|
|
'''
|
|
# If overwrite is set, for each group in the database that is NOT in
|
|
# the local list, delete it. When importing from a cloud inventory
|
|
# source attached to a specific group, only delete children of that
|
|
# group. Delete each group individually so signal handlers will run.
|
|
'''
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
groups_qs = self.inventory_source.groups.all()
|
|
# Build list of all group pks, remove those that should not be deleted.
|
|
del_group_pks = set(groups_qs.values_list('pk', flat=True))
|
|
all_group_names = list(self.all_group.all_groups.keys())
|
|
for offset in range(0, len(all_group_names), self._batch_size):
|
|
group_names = all_group_names[offset:(offset + self._batch_size)]
|
|
for group_pk in groups_qs.filter(name__in=group_names).values_list('pk', flat=True):
|
|
del_group_pks.discard(group_pk)
|
|
# Now delete all remaining groups in batches.
|
|
all_del_pks = sorted(list(del_group_pks))
|
|
for offset in range(0, len(all_del_pks), self._batch_size):
|
|
del_pks = all_del_pks[offset:(offset + self._batch_size)]
|
|
for group in groups_qs.filter(pk__in=del_pks):
|
|
group_name = group.name
|
|
with ignore_inventory_computed_fields():
|
|
group.delete()
|
|
logger.debug('Group "%s" deleted', group_name)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('group deletions took %d queries for %d groups',
|
|
len(connection.queries) - queries_before,
|
|
len(all_del_pks))
|
|
|
|
def _delete_group_children_and_hosts(self):
|
|
'''
|
|
Clear all invalid child relationships for groups and all invalid host
|
|
memberships. When importing from a cloud inventory source attached to
|
|
a specific group, only clear relationships for hosts and groups that
|
|
are beneath the inventory source group.
|
|
'''
|
|
# FIXME: Optimize performance!
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
group_group_count = 0
|
|
group_host_count = 0
|
|
db_groups = self.inventory_source.groups
|
|
# Set of all group names managed by this inventory source
|
|
all_source_group_names = frozenset(self.all_group.all_groups.keys())
|
|
# Set of all host pks managed by this inventory source
|
|
all_source_host_pks = self._existing_host_pks()
|
|
for db_group in db_groups.all():
|
|
# Delete child group relationships not present in imported data.
|
|
db_children = db_group.children
|
|
db_children_name_pk_map = dict(db_children.values_list('name', 'pk'))
|
|
# Exclude child groups from removal list if they were returned by
|
|
# the import, because this parent-child relationship has not changed
|
|
mem_children = self.all_group.all_groups[db_group.name].children
|
|
for mem_group in mem_children:
|
|
db_children_name_pk_map.pop(mem_group.name, None)
|
|
# Exclude child groups from removal list if they were not imported
|
|
# by this specific inventory source, because
|
|
# those relationships are outside of the dominion of this inventory source
|
|
other_source_group_names = set(db_children_name_pk_map.keys()) - all_source_group_names
|
|
for group_name in other_source_group_names:
|
|
db_children_name_pk_map.pop(group_name, None)
|
|
# Removal list is complete - now perform the removals
|
|
del_child_group_pks = list(set(db_children_name_pk_map.values()))
|
|
for offset in range(0, len(del_child_group_pks), self._batch_size):
|
|
child_group_pks = del_child_group_pks[offset:(offset + self._batch_size)]
|
|
for db_child in db_children.filter(pk__in=child_group_pks):
|
|
group_group_count += 1
|
|
db_group.children.remove(db_child)
|
|
logger.debug('Group "%s" removed from group "%s"',
|
|
db_child.name, db_group.name)
|
|
# FIXME: Inventory source group relationships
|
|
# Delete group/host relationships not present in imported data.
|
|
db_hosts = db_group.hosts
|
|
del_host_pks = set(db_hosts.values_list('pk', flat=True))
|
|
# Exclude child hosts from removal list if they were not imported
|
|
# by this specific inventory source, because
|
|
# those relationships are outside of the dominion of this inventory source
|
|
del_host_pks = del_host_pks & all_source_host_pks
|
|
# Exclude child hosts from removal list if they were returned by
|
|
# the import, because this group-host relationship has not changed
|
|
mem_hosts = self.all_group.all_groups[db_group.name].hosts
|
|
all_mem_host_names = [h.name for h in mem_hosts if not h.instance_id]
|
|
for offset in range(0, len(all_mem_host_names), self._batch_size):
|
|
mem_host_names = all_mem_host_names[offset:(offset + self._batch_size)]
|
|
for db_host_pk in db_hosts.filter(name__in=mem_host_names).values_list('pk', flat=True):
|
|
del_host_pks.discard(db_host_pk)
|
|
all_mem_instance_ids = [h.instance_id for h in mem_hosts if h.instance_id]
|
|
for offset in range(0, len(all_mem_instance_ids), self._batch_size):
|
|
mem_instance_ids = all_mem_instance_ids[offset:(offset + self._batch_size)]
|
|
for db_host_pk in db_hosts.filter(instance_id__in=mem_instance_ids).values_list('pk', flat=True):
|
|
del_host_pks.discard(db_host_pk)
|
|
all_db_host_pks = [v for k,v in self.db_instance_id_map.items() if k in all_mem_instance_ids]
|
|
for db_host_pk in all_db_host_pks:
|
|
del_host_pks.discard(db_host_pk)
|
|
# Removal list is complete - now perform the removals
|
|
del_host_pks = list(del_host_pks)
|
|
for offset in range(0, len(del_host_pks), self._batch_size):
|
|
del_pks = del_host_pks[offset:(offset + self._batch_size)]
|
|
for db_host in db_hosts.filter(pk__in=del_pks):
|
|
group_host_count += 1
|
|
if db_host not in db_group.hosts.all():
|
|
continue
|
|
db_group.hosts.remove(db_host)
|
|
logger.debug('Host "%s" removed from group "%s"',
|
|
db_host.name, db_group.name)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('group-group and group-host deletions took %d queries for %d relationships',
|
|
len(connection.queries) - queries_before,
|
|
group_group_count + group_host_count)
|
|
|
|
def _update_inventory(self):
|
|
'''
|
|
Update inventory variables from "all" group.
|
|
'''
|
|
# TODO: We disable variable overwrite here in case user-defined inventory variables get
|
|
# mangled. But we still need to figure out a better way of processing multiple inventory
|
|
# update variables mixing with each other.
|
|
all_obj = self.inventory
|
|
db_variables = all_obj.variables_dict
|
|
db_variables.update(self.all_group.variables)
|
|
if db_variables != all_obj.variables_dict:
|
|
all_obj.variables = json.dumps(db_variables)
|
|
all_obj.save(update_fields=['variables'])
|
|
logger.debug('Inventory variables updated from "all" group')
|
|
else:
|
|
logger.debug('Inventory variables unmodified')
|
|
|
|
def _create_update_groups(self):
|
|
'''
|
|
For each group in the local list, create it if it doesn't exist in the
|
|
database. Otherwise, update/replace database variables from the
|
|
imported data. Associate with the inventory source group if importing
|
|
from cloud inventory source.
|
|
'''
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
all_group_names = sorted(self.all_group.all_groups.keys())
|
|
root_group_names = set()
|
|
for k,v in self.all_group.all_groups.items():
|
|
if not v.parents:
|
|
root_group_names.add(k)
|
|
if len(v.parents) == 1 and v.parents[0].name == 'all':
|
|
root_group_names.add(k)
|
|
existing_group_names = set()
|
|
for offset in range(0, len(all_group_names), self._batch_size):
|
|
group_names = all_group_names[offset:(offset + self._batch_size)]
|
|
for group in self.inventory.groups.filter(name__in=group_names):
|
|
mem_group = self.all_group.all_groups[group.name]
|
|
db_variables = group.variables_dict
|
|
if self.overwrite_vars:
|
|
db_variables = mem_group.variables
|
|
else:
|
|
db_variables.update(mem_group.variables)
|
|
if db_variables != group.variables_dict:
|
|
group.variables = json.dumps(db_variables)
|
|
group.save(update_fields=['variables'])
|
|
if self.overwrite_vars:
|
|
logger.debug('Group "%s" variables replaced', group.name)
|
|
else:
|
|
logger.debug('Group "%s" variables updated', group.name)
|
|
else:
|
|
logger.debug('Group "%s" variables unmodified', group.name)
|
|
existing_group_names.add(group.name)
|
|
self._batch_add_m2m(self.inventory_source.groups, group)
|
|
for group_name in all_group_names:
|
|
if group_name in existing_group_names:
|
|
continue
|
|
mem_group = self.all_group.all_groups[group_name]
|
|
group_desc = mem_group.variables.pop('_awx_description', 'imported')
|
|
group = self.inventory.groups.update_or_create(
|
|
name=group_name,
|
|
defaults={
|
|
'variables':json.dumps(mem_group.variables),
|
|
'description':group_desc
|
|
}
|
|
)[0]
|
|
logger.debug('Group "%s" added', group.name)
|
|
self._batch_add_m2m(self.inventory_source.groups, group)
|
|
self._batch_add_m2m(self.inventory_source.groups, flush=True)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('group updates took %d queries for %d groups',
|
|
len(connection.queries) - queries_before,
|
|
len(self.all_group.all_groups))
|
|
|
|
def _update_db_host_from_mem_host(self, db_host, mem_host):
|
|
# Update host variables.
|
|
db_variables = db_host.variables_dict
|
|
if self.overwrite_vars:
|
|
db_variables = mem_host.variables
|
|
else:
|
|
db_variables.update(mem_host.variables)
|
|
update_fields = []
|
|
if db_variables != db_host.variables_dict:
|
|
db_host.variables = json.dumps(db_variables)
|
|
update_fields.append('variables')
|
|
# Update host enabled flag.
|
|
enabled = self._get_enabled(mem_host.variables)
|
|
if enabled is not None and db_host.enabled != enabled:
|
|
db_host.enabled = enabled
|
|
update_fields.append('enabled')
|
|
# Update host name.
|
|
if mem_host.name != db_host.name:
|
|
old_name = db_host.name
|
|
db_host.name = mem_host.name
|
|
update_fields.append('name')
|
|
# Update host instance_id.
|
|
instance_id = self._get_instance_id(mem_host.variables)
|
|
if instance_id != db_host.instance_id:
|
|
old_instance_id = db_host.instance_id
|
|
db_host.instance_id = instance_id
|
|
update_fields.append('instance_id')
|
|
# Update host and display message(s) on what changed.
|
|
if update_fields:
|
|
db_host.save(update_fields=update_fields)
|
|
if 'name' in update_fields:
|
|
logger.debug('Host renamed from "%s" to "%s"', old_name, mem_host.name)
|
|
if 'instance_id' in update_fields:
|
|
if old_instance_id:
|
|
logger.debug('Host "%s" instance_id updated', mem_host.name)
|
|
else:
|
|
logger.debug('Host "%s" instance_id added', mem_host.name)
|
|
if 'variables' in update_fields:
|
|
if self.overwrite_vars:
|
|
logger.debug('Host "%s" variables replaced', mem_host.name)
|
|
else:
|
|
logger.debug('Host "%s" variables updated', mem_host.name)
|
|
else:
|
|
logger.debug('Host "%s" variables unmodified', mem_host.name)
|
|
if 'enabled' in update_fields:
|
|
if enabled:
|
|
logger.debug('Host "%s" is now enabled', mem_host.name)
|
|
else:
|
|
logger.debug('Host "%s" is now disabled', mem_host.name)
|
|
self._batch_add_m2m(self.inventory_source.hosts, db_host)
|
|
|
|
def _create_update_hosts(self):
|
|
'''
|
|
For each host in the local list, create it if it doesn't exist in the
|
|
database. Otherwise, update/replace database variables from the
|
|
imported data. Associate with the inventory source group if importing
|
|
from cloud inventory source.
|
|
'''
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
host_pks_updated = set()
|
|
mem_host_pk_map = {}
|
|
mem_host_instance_id_map = {}
|
|
mem_host_name_map = {}
|
|
mem_host_names_to_update = set(self.all_group.all_hosts.keys())
|
|
for k,v in self.all_group.all_hosts.items():
|
|
mem_host_name_map[k] = v
|
|
instance_id = self._get_instance_id(v.variables)
|
|
if instance_id in self.db_instance_id_map:
|
|
mem_host_pk_map[self.db_instance_id_map[instance_id]] = v
|
|
elif instance_id:
|
|
mem_host_instance_id_map[instance_id] = v
|
|
|
|
# Update all existing hosts where we know the PK based on instance_id.
|
|
all_host_pks = sorted(mem_host_pk_map.keys())
|
|
for offset in range(0, len(all_host_pks), self._batch_size):
|
|
host_pks = all_host_pks[offset:(offset + self._batch_size)]
|
|
for db_host in self.inventory.hosts.filter( pk__in=host_pks):
|
|
if db_host.pk in host_pks_updated:
|
|
continue
|
|
mem_host = mem_host_pk_map[db_host.pk]
|
|
self._update_db_host_from_mem_host(db_host, mem_host)
|
|
host_pks_updated.add(db_host.pk)
|
|
mem_host_names_to_update.discard(mem_host.name)
|
|
|
|
# Update all existing hosts where we know the instance_id.
|
|
all_instance_ids = sorted(mem_host_instance_id_map.keys())
|
|
for offset in range(0, len(all_instance_ids), self._batch_size):
|
|
instance_ids = all_instance_ids[offset:(offset + self._batch_size)]
|
|
for db_host in self.inventory.hosts.filter( instance_id__in=instance_ids):
|
|
if db_host.pk in host_pks_updated:
|
|
continue
|
|
mem_host = mem_host_instance_id_map[db_host.instance_id]
|
|
self._update_db_host_from_mem_host(db_host, mem_host)
|
|
host_pks_updated.add(db_host.pk)
|
|
mem_host_names_to_update.discard(mem_host.name)
|
|
|
|
# Update all existing hosts by name.
|
|
all_host_names = sorted(mem_host_name_map.keys())
|
|
for offset in range(0, len(all_host_names), self._batch_size):
|
|
host_names = all_host_names[offset:(offset + self._batch_size)]
|
|
for db_host in self.inventory.hosts.filter( name__in=host_names):
|
|
if db_host.pk in host_pks_updated:
|
|
continue
|
|
mem_host = mem_host_name_map[db_host.name]
|
|
self._update_db_host_from_mem_host(db_host, mem_host)
|
|
host_pks_updated.add(db_host.pk)
|
|
mem_host_names_to_update.discard(mem_host.name)
|
|
|
|
# Create any new hosts.
|
|
for mem_host_name in sorted(mem_host_names_to_update):
|
|
mem_host = self.all_group.all_hosts[mem_host_name]
|
|
import_vars = mem_host.variables
|
|
host_desc = import_vars.pop('_awx_description', 'imported')
|
|
host_attrs = dict(variables=json.dumps(import_vars), description=host_desc)
|
|
enabled = self._get_enabled(mem_host.variables)
|
|
if enabled is not None:
|
|
host_attrs['enabled'] = enabled
|
|
if self.instance_id_var:
|
|
instance_id = self._get_instance_id(mem_host.variables)
|
|
host_attrs['instance_id'] = instance_id
|
|
try:
|
|
sanitize_jinja(mem_host_name)
|
|
except ValueError as e:
|
|
raise ValueError(str(e) + ': {}'.format(mem_host_name))
|
|
db_host = self.inventory.hosts.update_or_create(name=mem_host_name, defaults=host_attrs)[0]
|
|
if enabled is False:
|
|
logger.debug('Host "%s" added (disabled)', mem_host_name)
|
|
else:
|
|
logger.debug('Host "%s" added', mem_host_name)
|
|
self._batch_add_m2m(self.inventory_source.hosts, db_host)
|
|
|
|
self._batch_add_m2m(self.inventory_source.hosts, flush=True)
|
|
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('host updates took %d queries for %d hosts',
|
|
len(connection.queries) - queries_before,
|
|
len(self.all_group.all_hosts))
|
|
|
|
@transaction.atomic
|
|
def _create_update_group_children(self):
|
|
'''
|
|
For each imported group, create all parent-child group relationships.
|
|
'''
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
all_group_names = sorted([k for k,v in self.all_group.all_groups.items() if v.children])
|
|
group_group_count = 0
|
|
for offset in range(0, len(all_group_names), self._batch_size):
|
|
group_names = all_group_names[offset:(offset + self._batch_size)]
|
|
for db_group in self.inventory.groups.filter(name__in=group_names):
|
|
mem_group = self.all_group.all_groups[db_group.name]
|
|
group_group_count += len(mem_group.children)
|
|
all_child_names = sorted([g.name for g in mem_group.children])
|
|
for offset2 in range(0, len(all_child_names), self._batch_size):
|
|
child_names = all_child_names[offset2:(offset2 + self._batch_size)]
|
|
db_children_qs = self.inventory.groups.filter(name__in=child_names)
|
|
for db_child in db_children_qs.filter(children__id=db_group.id):
|
|
logger.debug('Group "%s" already child of group "%s"', db_child.name, db_group.name)
|
|
for db_child in db_children_qs.exclude(children__id=db_group.id):
|
|
self._batch_add_m2m(db_group.children, db_child)
|
|
logger.debug('Group "%s" added as child of "%s"', db_child.name, db_group.name)
|
|
self._batch_add_m2m(db_group.children, flush=True)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('Group-group updates took %d queries for %d group-group relationships',
|
|
len(connection.queries) - queries_before, group_group_count)
|
|
|
|
@transaction.atomic
|
|
def _create_update_group_hosts(self):
|
|
# For each host in a mem group, add it to the parent(s) to which it
|
|
# belongs.
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
all_group_names = sorted([k for k,v in self.all_group.all_groups.items() if v.hosts])
|
|
group_host_count = 0
|
|
for offset in range(0, len(all_group_names), self._batch_size):
|
|
group_names = all_group_names[offset:(offset + self._batch_size)]
|
|
for db_group in self.inventory.groups.filter(name__in=group_names):
|
|
mem_group = self.all_group.all_groups[db_group.name]
|
|
group_host_count += len(mem_group.hosts)
|
|
all_host_names = sorted([h.name for h in mem_group.hosts if not h.instance_id])
|
|
for offset2 in range(0, len(all_host_names), self._batch_size):
|
|
host_names = all_host_names[offset2:(offset2 + self._batch_size)]
|
|
db_hosts_qs = self.inventory.hosts.filter(name__in=host_names)
|
|
for db_host in db_hosts_qs.filter(groups__id=db_group.id):
|
|
logger.debug('Host "%s" already in group "%s"', db_host.name, db_group.name)
|
|
for db_host in db_hosts_qs.exclude(groups__id=db_group.id):
|
|
self._batch_add_m2m(db_group.hosts, db_host)
|
|
logger.debug('Host "%s" added to group "%s"', db_host.name, db_group.name)
|
|
all_instance_ids = sorted([h.instance_id for h in mem_group.hosts if h.instance_id])
|
|
for offset2 in range(0, len(all_instance_ids), self._batch_size):
|
|
instance_ids = all_instance_ids[offset2:(offset2 + self._batch_size)]
|
|
db_hosts_qs = self.inventory.hosts.filter(instance_id__in=instance_ids)
|
|
for db_host in db_hosts_qs.filter(groups__id=db_group.id):
|
|
logger.debug('Host "%s" already in group "%s"', db_host.name, db_group.name)
|
|
for db_host in db_hosts_qs.exclude(groups__id=db_group.id):
|
|
self._batch_add_m2m(db_group.hosts, db_host)
|
|
logger.debug('Host "%s" added to group "%s"', db_host.name, db_group.name)
|
|
self._batch_add_m2m(db_group.hosts, flush=True)
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('Group-host updates took %d queries for %d group-host relationships',
|
|
len(connection.queries) - queries_before, group_host_count)
|
|
|
|
def load_into_database(self):
|
|
'''
|
|
Load inventory from in-memory groups to the database, overwriting or
|
|
merging as appropriate.
|
|
'''
|
|
# FIXME: Attribute changes to superuser?
|
|
# Perform __in queries in batches (mainly for unit tests using SQLite).
|
|
self._batch_size = 500
|
|
self._build_db_instance_id_map()
|
|
self._build_mem_instance_id_map()
|
|
if self.overwrite:
|
|
self._delete_hosts()
|
|
self._delete_groups()
|
|
self._delete_group_children_and_hosts()
|
|
self._update_inventory()
|
|
self._create_update_groups()
|
|
self._create_update_hosts()
|
|
self._create_update_group_children()
|
|
self._create_update_group_hosts()
|
|
|
|
def remote_tower_license_compare(self, local_license_type):
|
|
# this requires https://github.com/ansible/ansible/pull/52747
|
|
source_vars = self.all_group.variables
|
|
remote_license_type = source_vars.get('tower_metadata', {}).get('license_type', None)
|
|
if remote_license_type is None:
|
|
raise PermissionDenied('Unexpected Error: Tower inventory plugin missing needed metadata!')
|
|
if local_license_type != remote_license_type:
|
|
raise PermissionDenied('Tower server licenses must match: source: {} local: {}'.format(
|
|
remote_license_type, local_license_type
|
|
))
|
|
|
|
def check_license(self):
|
|
license_info = get_licenser().validate()
|
|
local_license_type = license_info.get('license_type', 'UNLICENSED')
|
|
if local_license_type == 'UNLICENSED':
|
|
logger.error(LICENSE_NON_EXISTANT_MESSAGE)
|
|
raise PermissionDenied('No license found!')
|
|
elif local_license_type == 'open':
|
|
return
|
|
instance_count = license_info.get('instance_count', 0)
|
|
free_instances = license_info.get('free_instances', 0)
|
|
time_remaining = license_info.get('time_remaining', 0)
|
|
hard_error = license_info.get('trial', False) is True or license_info['instance_count'] == 10
|
|
new_count = Host.objects.active_count()
|
|
if time_remaining <= 0:
|
|
if hard_error:
|
|
logger.error(LICENSE_EXPIRED_MESSAGE)
|
|
raise PermissionDenied("License has expired!")
|
|
else:
|
|
logger.warning(LICENSE_EXPIRED_MESSAGE)
|
|
# special check for tower-type inventory sources
|
|
# but only if running the plugin
|
|
TOWER_SOURCE_FILES = ['tower.yml', 'tower.yaml']
|
|
if self.inventory_source.source == 'tower' and any(f in self.inventory_source.source_path for f in TOWER_SOURCE_FILES):
|
|
# only if this is the 2nd call to license check, we cannot compare before running plugin
|
|
if hasattr(self, 'all_group'):
|
|
self.remote_tower_license_compare(local_license_type)
|
|
if free_instances < 0:
|
|
d = {
|
|
'new_count': new_count,
|
|
'instance_count': instance_count,
|
|
}
|
|
if hard_error:
|
|
logger.error(LICENSE_MESSAGE % d)
|
|
raise PermissionDenied('License count exceeded!')
|
|
else:
|
|
logger.warning(LICENSE_MESSAGE % d)
|
|
|
|
def check_org_host_limit(self):
|
|
license_info = get_licenser().validate()
|
|
if license_info.get('license_type', 'UNLICENSED') == 'open':
|
|
return
|
|
|
|
org = self.inventory.organization
|
|
if org is None or org.max_hosts == 0:
|
|
return
|
|
|
|
active_count = Host.objects.org_active_count(org.id)
|
|
if active_count > org.max_hosts:
|
|
raise PermissionDenied('Host limit for organization exceeded!')
|
|
|
|
def mark_license_failure(self, save=True):
|
|
self.inventory_update.license_error = True
|
|
self.inventory_update.save(update_fields=['license_error'])
|
|
|
|
def mark_org_limits_failure(self, save=True):
|
|
self.inventory_update.org_host_limit_error = True
|
|
self.inventory_update.save(update_fields=['org_host_limit_error'])
|
|
|
|
def handle(self, *args, **options):
|
|
# Load inventory and related objects from database.
|
|
inventory_name = options.get('inventory_name', None)
|
|
inventory_id = options.get('inventory_id', None)
|
|
if inventory_name and inventory_id:
|
|
raise CommandError('--inventory-name and --inventory-id are mutually exclusive')
|
|
elif not inventory_name and not inventory_id:
|
|
raise CommandError('--inventory-name or --inventory-id is required')
|
|
|
|
with advisory_lock('inventory_{}_import'.format(inventory_id)):
|
|
# Obtain rest of the options needed to run update
|
|
raw_source = options.get('source', None)
|
|
if not raw_source:
|
|
raise CommandError('--source is required')
|
|
verbosity = int(options.get('verbosity', 1))
|
|
self.set_logging_level(verbosity)
|
|
venv_path = options.get('venv', None)
|
|
|
|
# Load inventory object based on name or ID.
|
|
if inventory_id:
|
|
q = dict(id=inventory_id)
|
|
else:
|
|
q = dict(name=inventory_name)
|
|
try:
|
|
inventory = Inventory.objects.get(**q)
|
|
except Inventory.DoesNotExist:
|
|
raise CommandError('Inventory with %s = %s cannot be found' % list(q.items())[0])
|
|
except Inventory.MultipleObjectsReturned:
|
|
raise CommandError('Inventory with %s = %s returned multiple results' % list(q.items())[0])
|
|
logger.info('Updating inventory %d: %s' % (inventory.pk, inventory.name))
|
|
|
|
|
|
# Create ad-hoc inventory source and inventory update objects
|
|
with ignore_inventory_computed_fields():
|
|
source = Command.get_source_absolute_path(raw_source)
|
|
|
|
inventory_source, created = InventorySource.objects.get_or_create(
|
|
inventory=inventory,
|
|
source='file',
|
|
source_path=os.path.abspath(source),
|
|
overwrite=bool(options.get('overwrite', False)),
|
|
overwrite_vars=bool(options.get('overwrite_vars', False)),
|
|
)
|
|
inventory_update = inventory_source.create_inventory_update(
|
|
_eager_fields=dict(
|
|
job_args=json.dumps(sys.argv),
|
|
job_env=dict(os.environ.items()),
|
|
job_cwd=os.getcwd())
|
|
)
|
|
|
|
data = AnsibleInventoryLoader(
|
|
source=source, venv_path=venv_path, verbosity=verbosity
|
|
).load()
|
|
|
|
logger.debug('Finished loading from source: %s', source)
|
|
|
|
status, tb, exc = 'error', '', None
|
|
try:
|
|
self.perform_update(options, data, inventory_update)
|
|
status = 'successful'
|
|
except Exception as e:
|
|
exc = e
|
|
if isinstance(e, KeyboardInterrupt):
|
|
status = 'canceled'
|
|
else:
|
|
tb = traceback.format_exc()
|
|
|
|
with ignore_inventory_computed_fields():
|
|
inventory_update = InventoryUpdate.objects.get(pk=inventory_update.pk)
|
|
inventory_update.result_traceback = tb
|
|
inventory_update.status = status
|
|
inventory_update.save(update_fields=['status', 'result_traceback'])
|
|
inventory_source.status = status
|
|
inventory_source.save(update_fields=['status'])
|
|
|
|
if exc:
|
|
logger.error(str(exc))
|
|
|
|
if exc:
|
|
if isinstance(exc, CommandError):
|
|
sys.exit(1)
|
|
raise exc
|
|
|
|
def perform_update(self, options, data, inventory_update):
|
|
"""Shared method for both awx-manage CLI updates and inventory updates
|
|
from the tasks system.
|
|
|
|
This saves the inventory data to the database, calling load_into_database
|
|
but also wraps that method in a host of options processing
|
|
"""
|
|
# outside of normal options, these are needed as part of programatic interface
|
|
self.inventory = inventory_update.inventory
|
|
self.inventory_source = inventory_update.inventory_source
|
|
self.inventory_update = inventory_update
|
|
|
|
# the update options, could be parser object or dict
|
|
self.overwrite = bool(options.get('overwrite', False))
|
|
self.overwrite_vars = bool(options.get('overwrite_vars', False))
|
|
self.enabled_var = options.get('enabled_var', None)
|
|
self.enabled_value = options.get('enabled_value', None)
|
|
self.group_filter = options.get('group_filter', None) or r'^.+$'
|
|
self.host_filter = options.get('host_filter', None) or r'^.+$'
|
|
self.exclude_empty_groups = bool(options.get('exclude_empty_groups', False))
|
|
self.instance_id_var = options.get('instance_id_var', None)
|
|
|
|
try:
|
|
self.group_filter_re = re.compile(self.group_filter)
|
|
except re.error:
|
|
raise CommandError('invalid regular expression for --group-filter')
|
|
try:
|
|
self.host_filter_re = re.compile(self.host_filter)
|
|
except re.error:
|
|
raise CommandError('invalid regular expression for --host-filter')
|
|
|
|
begin = time.time()
|
|
|
|
# Since perform_update can be invoked either through the awx-manage CLI
|
|
# or from the task system, we need to create a new lock at this level
|
|
# (even though inventory_import.Command.handle -- which calls
|
|
# perform_update -- has its own lock, inventory_ID_import)
|
|
with advisory_lock('inventory_{}_perform_update'.format(self.inventory.id)):
|
|
|
|
try:
|
|
self.check_license()
|
|
except PermissionDenied as e:
|
|
self.mark_license_failure(save=True)
|
|
raise e
|
|
|
|
try:
|
|
# Check the per-org host limits
|
|
self.check_org_host_limit()
|
|
except PermissionDenied as e:
|
|
self.mark_org_limits_failure(save=True)
|
|
raise e
|
|
|
|
if settings.SQL_DEBUG:
|
|
queries_before = len(connection.queries)
|
|
|
|
# Update inventory update for this command line invocation.
|
|
with ignore_inventory_computed_fields():
|
|
# TODO: move this to before perform_update
|
|
iu = self.inventory_update
|
|
if iu.status != 'running':
|
|
with transaction.atomic():
|
|
self.inventory_update.status = 'running'
|
|
self.inventory_update.save()
|
|
|
|
logger.info('Processing JSON output...')
|
|
inventory = MemInventory(
|
|
group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
|
|
inventory = dict_to_mem_data(data, inventory=inventory)
|
|
|
|
logger.info('Loaded %d groups, %d hosts', len(inventory.all_group.all_groups),
|
|
len(inventory.all_group.all_hosts))
|
|
|
|
if self.exclude_empty_groups:
|
|
inventory.delete_empty_groups()
|
|
|
|
self.all_group = inventory.all_group
|
|
|
|
if settings.DEBUG:
|
|
# depending on inventory source, this output can be
|
|
# *exceedingly* verbose - crawling a deeply nested
|
|
# inventory/group data structure and printing metadata about
|
|
# each host and its memberships
|
|
#
|
|
# it's easy for this scale of data to overwhelm pexpect,
|
|
# (and it's likely only useful for purposes of debugging the
|
|
# actual inventory import code), so only print it if we have to:
|
|
# https://github.com/ansible/ansible-tower/issues/7414#issuecomment-321615104
|
|
self.all_group.debug_tree()
|
|
|
|
with batch_role_ancestor_rebuilding():
|
|
# If using with transaction.atomic() with try ... catch,
|
|
# with transaction.atomic() must be inside the try section of the code as per Django docs
|
|
try:
|
|
# Ensure that this is managed as an atomic SQL transaction,
|
|
# and thus properly rolled back if there is an issue.
|
|
with transaction.atomic():
|
|
# Merge/overwrite inventory into database.
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('loading into database...')
|
|
with ignore_inventory_computed_fields():
|
|
if getattr(settings, 'ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC', True):
|
|
self.load_into_database()
|
|
else:
|
|
with disable_activity_stream():
|
|
self.load_into_database()
|
|
if settings.SQL_DEBUG:
|
|
queries_before2 = len(connection.queries)
|
|
self.inventory.update_computed_fields()
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('update computed fields took %d queries',
|
|
len(connection.queries) - queries_before2)
|
|
|
|
# Check if the license is valid.
|
|
# If the license is not valid, a CommandError will be thrown,
|
|
# and inventory update will be marked as invalid.
|
|
# with transaction.atomic() will roll back the changes.
|
|
license_fail = True
|
|
self.check_license()
|
|
|
|
# Check the per-org host limits
|
|
license_fail = False
|
|
self.check_org_host_limit()
|
|
except PermissionDenied as e:
|
|
if license_fail:
|
|
self.mark_license_failure(save=True)
|
|
else:
|
|
self.mark_org_limits_failure(save=True)
|
|
raise e
|
|
|
|
if settings.SQL_DEBUG:
|
|
logger.warning('Inventory import completed for %s in %0.1fs',
|
|
self.inventory_source.name, time.time() - begin)
|
|
else:
|
|
logger.info('Inventory import completed for %s in %0.1fs',
|
|
self.inventory_source.name, time.time() - begin)
|
|
|
|
# If we're in debug mode, then log the queries and time
|
|
# used to do the operation.
|
|
if settings.SQL_DEBUG:
|
|
queries_this_import = connection.queries[queries_before:]
|
|
sqltime = sum(float(x['time']) for x in queries_this_import)
|
|
logger.warning('Inventory import required %d queries '
|
|
'taking %0.3fs', len(queries_this_import),
|
|
sqltime)
|