Merge pull request #18 from ctrliq/devel

Sync to Upstream 23.7.0

.github/workflows/promote.yml

@@ -69,7 +69,7 @@ jobs:
 #      - name: Build awxkit and upload to pypi
 #        run: |
 #          git reset --hard
-#          cd awxkit && python3 setup.py bdist_wheel
+#          cd awxkit && python3 setup.py sdist bdist_wheel
 #          twine upload \
 #            -r ${{ env.pypi_repo }} \
 #            -u ${{ secrets.PYPI_USERNAME }} \

awx/api/generics.py

@@ -30,8 +30,8 @@
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
-from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
-from ansible_base.utils.models import get_all_field_names
+from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
+from ansible_base.lib.utils.models import get_all_field_names
 
 # AWX
 from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
@@ -91,7 +91,7 @@ def post(self, request, *args, **kwargs):
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         if request.user.is_authenticated:
             logger.info(smart_str(u"User {} logged in from {}".format(self.request.user.username, request.META.get('REMOTE_ADDR', None))))
-            ret.set_cookie('userLoggedIn', 'true')
+            ret.set_cookie('userLoggedIn', 'true', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False))
             ret.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
 
             return ret
@@ -107,7 +107,7 @@ def dispatch(self, request, *args, **kwargs):
         original_user = getattr(request, 'user', None)
         ret = super(LoggedLogoutView, self).dispatch(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
-        ret.set_cookie('userLoggedIn', 'false')
+        ret.set_cookie('userLoggedIn', 'false', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False))
         if (not current_user or not getattr(current_user, 'pk', True)) and current_user != original_user:
             logger.info("User {} logged out.".format(original_user.username))
         return ret

awx/api/serializers.py

@@ -43,7 +43,7 @@
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
-from ansible_base.utils.models import get_type_for_model
+from ansible_base.lib.utils.models import get_type_for_model
 
 # AWX
 from awx.main.access import get_user_capabilities

awx/api/urls/webhooks.py

@@ -1,10 +1,11 @@
 from django.urls import re_path
 
-from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver
+from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver, BitbucketDcWebhookReceiver
 
 
 urlpatterns = [
     re_path(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
     re_path(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
     re_path(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+    re_path(r'^bitbucket_dc/$', BitbucketDcWebhookReceiver.as_view(), name='webhook_receiver_bitbucket_dc'),
 ]

awx/api/views/webhooks.py

@@ -1,4 +1,4 @@
-from hashlib import sha1
+from hashlib import sha1, sha256
 import hmac
 import logging
 import urllib.parse
@@ -99,14 +99,31 @@ def get_event_ref(self):
     def get_signature(self):
         raise NotImplementedError
 
+    def must_check_signature(self):
+        return True
+
+    def is_ignored_request(self):
+        return False
+
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
+        if not self.must_check_signature():
+            logger.debug("skipping signature validation")
+            return
+
+        hash_alg, expected_digest = self.get_signature()
+        if hash_alg == 'sha1':
+            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        elif hash_alg == 'sha256':
+            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha256)
+        else:
+            logger.debug("Unsupported signature type, supported: sha1, sha256, received: {}".format(hash_alg))
+            raise PermissionDenied
 
-        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
-        logger.debug("header signature: %s", self.get_signature())
+        logger.debug("header signature: %s", expected_digest)
         logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
-        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), expected_digest):
             raise PermissionDenied
 
     @csrf_exempt
@@ -118,6 +135,10 @@ def post(self, request, *args, **kwargs):
         obj = self.get_object()
         self.check_signature(obj)
 
+        if self.is_ignored_request():
+            # This was an ignored request type (e.g. ping), don't act on it
+            return Response({'message': _("Webhook ignored")}, status=status.HTTP_200_OK)
+
         event_type = self.get_event_type()
         event_guid = self.get_event_guid()
         event_ref = self.get_event_ref()
@@ -186,7 +207,7 @@ def get_signature(self):
         if hash_alg != 'sha1':
             logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
             raise PermissionDenied
-        return force_bytes(signature)
+        return hash_alg, force_bytes(signature)
 
 
 class GitlabWebhookReceiver(WebhookReceiverBase):
@@ -214,15 +235,73 @@ def get_event_status_api(self):
 
         return "{}://{}/api/v4/projects/{}/statuses/{}".format(parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
 
-    def get_signature(self):
-        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
-
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
 
+        token_from_request = force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
         # GitLab only returns the secret token, not an hmac hash.  Use
         # the hmac `compare_digest` helper function to prevent timing
         # analysis by attackers.
-        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), token_from_request):
             raise PermissionDenied
+
+
+class BitbucketDcWebhookReceiver(WebhookReceiverBase):
+    service = 'bitbucket_dc'
+
+    ref_keys = {
+        'repo:refs_changed': 'changes.0.toHash',
+        'mirror:repo_synchronized': 'changes.0.toHash',
+        'pr:opened': 'pullRequest.toRef.latestCommit',
+        'pr:from_ref_updated': 'pullRequest.toRef.latestCommit',
+        'pr:modified': 'pullRequest.toRef.latestCommit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_EVENT_KEY')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_REQUEST_ID')
+
+    def get_event_status_api(self):
+        # https://<bitbucket-base-url>/rest/build-status/1.0/commits/<commit-hash>
+        if self.get_event_type() not in self.ref_keys.keys():
+            return
+        if self.get_event_ref() is None:
+            return
+        any_url = None
+        if 'actor' in self.request.data:
+            any_url = self.request.data['actor'].get('links', {}).get('self')
+        if any_url is None and 'repository' in self.request.data:
+            any_url = self.request.data['repository'].get('links', {}).get('self')
+        if any_url is None:
+            return
+        any_url = any_url[0].get('href')
+        if any_url is None:
+            return
+        parsed = urllib.parse.urlparse(any_url)
+
+        return "{}://{}/rest/build-status/1.0/commits/{}".format(parsed.scheme, parsed.netloc, self.get_event_ref())
+
+    def is_ignored_request(self):
+        return self.get_event_type() not in [
+            'repo:refs_changed',
+            'mirror:repo_synchronized',
+            'pr:opened',
+            'pr:from_ref_updated',
+            'pr:modified',
+        ]
+
+    def must_check_signature(self):
+        # Bitbucket does not sign ping requests...
+        return self.get_event_type() != 'diagnostics:ping'
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        return hash_alg, force_bytes(signature)

awx/conf/models.py

@@ -7,7 +7,7 @@
 # Django
 from django.db import models
 
-from ansible_base.utils.models import prevent_search
+from ansible_base.lib.utils.models import prevent_search
 
 # AWX
 from awx.main.models.base import CreatedModifiedModel

awx/main/access.py

@@ -20,7 +20,7 @@
 # Django OAuth Toolkit
 from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
 
-from ansible_base.utils.validation import to_python_boolean
+from ansible_base.lib.utils.validation import to_python_boolean
 
 # AWX
 from awx.main.utils import (

awx/main/credential_plugins/aim.py

@@ -58,7 +58,7 @@
             'id': 'object_property',
             'label': _('Object Property'),
             'type': 'string',
-            'help_text': _('The property of the object to return. Default: Content Ex: Username, Address, etc.'),
+            'help_text': _('The property of the object to return. Available properties: Username, Password and Address.'),
         },
         {
             'id': 'reason',
@@ -111,8 +111,12 @@ def aim_backend(**kwargs):
         object_property = 'Content'
     elif object_property.lower() == 'username':
         object_property = 'UserName'
+    elif object_property.lower() == 'password':
+        object_property = 'Content'
+    elif object_property.lower() == 'address':
+        object_property = 'Address'
     elif object_property not in res:
-        raise KeyError('Property {} not found in object'.format(object_property))
+        raise KeyError('Property {} not found in object, available properties: Username, Password and Address'.format(object_property))
     else:
         object_property = object_property.capitalize()
 

awx/main/credential_plugins/hashivault.py

@@ -87,6 +87,20 @@
                 ' see https://www.vaultproject.io/docs/auth/kubernetes#configuration'
             ),
         },
+        {
+            'id': 'username',
+            'label': _('Username'),
+            'type': 'string',
+            'secret': False,
+            'help_text': _('Username for user authentication.'),
+        },
+        {
+            'id': 'password',
+            'label': _('Password'),
+            'type': 'string',
+            'secret': True,
+            'help_text': _('Password for user authentication.'),
+        },
         {
             'id': 'default_auth_path',
             'label': _('Path to Auth'),
@@ -185,21 +199,25 @@
 
 def handle_auth(**kwargs):
     token = None
-
     if kwargs.get('token'):
         token = kwargs['token']
+    elif kwargs.get('username') and kwargs.get('password'):
+        token = method_auth(**kwargs, auth_param=userpass_auth(**kwargs))
     elif kwargs.get('role_id') and kwargs.get('secret_id'):
         token = method_auth(**kwargs, auth_param=approle_auth(**kwargs))
     elif kwargs.get('kubernetes_role'):
         token = method_auth(**kwargs, auth_param=kubernetes_auth(**kwargs))
     elif kwargs.get('client_cert_public') and kwargs.get('client_cert_private'):
         token = method_auth(**kwargs, auth_param=client_cert_auth(**kwargs))
     else:
-        raise Exception('Either a token or AppRole, Kubernetes, or TLS authentication parameters must be set')
-
+        raise Exception('Token, Username/Password, AppRole, Kubernetes, or TLS authentication parameters must be set')
     return token
 
 
+def userpass_auth(**kwargs):
+    return {'username': kwargs['username'], 'password': kwargs['password']}
+
+
 def approle_auth(**kwargs):
     return {'role_id': kwargs['role_id'], 'secret_id': kwargs['secret_id']}
 
@@ -227,11 +245,14 @@ def method_auth(**kwargs):
     cacert = kwargs.get('cacert', None)
 
     sess = requests.Session()
+    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
 
     # Namespace support
     if kwargs.get('namespace'):
         sess.headers['X-Vault-Namespace'] = kwargs['namespace']
     request_url = '/'.join([url, 'auth', auth_path, 'login']).rstrip('/')
+    if kwargs['auth_param'].get('username'):
+        request_url = request_url + '/' + (kwargs['username'])
     with CertFiles(cacert) as cert:
         request_kwargs['verify'] = cert
         # TLS client certificate support
@@ -263,6 +284,7 @@ def kv_backend(**kwargs):
     }
 
     sess = requests.Session()
+    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
     # Compatibility header for older installs of Hashicorp Vault
     sess.headers['X-Vault-Token'] = token
@@ -333,6 +355,7 @@ def ssh_backend(**kwargs):
         request_kwargs['json']['valid_principals'] = kwargs['valid_principals']
 
     sess = requests.Session()
+    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
     if kwargs.get('namespace'):
         sess.headers['X-Vault-Namespace'] = kwargs['namespace']

awx/main/dispatch/__init__.py

@@ -93,6 +93,22 @@ def close(self):
         self.conn.close()
 
 
+def create_listener_connection():
+    conf = settings.DATABASES['default'].copy()
+    conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
+    # Modify the application name to distinguish from other connections the process might use
+    conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
+
+    # Apply overrides specifically for the listener connection
+    for k, v in settings.LISTENER_DATABASES.get('default', {}).items():
+        conf[k] = v
+    for k, v in settings.LISTENER_DATABASES.get('default', {}).get('OPTIONS', {}).items():
+        conf['OPTIONS'][k] = v
+
+    connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} password={conf['PASSWORD']} port={conf['PORT']}"
+    return psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
+
+
 @contextmanager
 def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
@@ -106,12 +122,7 @@ def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
 
     if new_connection:
-        conf = settings.DATABASES['default'].copy()
-        conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
-        # Modify the application name to distinguish from other connections the process might use
-        conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
-        connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} password={conf['PASSWORD']} port={conf['PORT']}"
-        conn = psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
+        conn = create_listener_connection()
     else:
         if pg_connection.connection is None:
             pg_connection.connect()

awx/main/dispatch/worker/base.py

@@ -214,7 +214,10 @@ def run_periodic_tasks(self):
                 # bypasses pg_notify for scheduled tasks
                 self.dispatch_task(body)
 
-        self.pg_is_down = False
+        if self.pg_is_down:
+            logger.info('Dispatcher listener connection established')
+            self.pg_is_down = False
+
         self.listen_start = time.time()
 
         return self.scheduler.time_until_next_run()