tests: refactor test harness to split apart a single file (#21391)

* re-instate previously flaky test

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* break up big file

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* move geoip data to subdir

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* i am but a weak man

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix ldap disconnect in testing

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* account for mismatched uid due to test server process

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

---------

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

Makefile

@@ -144,8 +144,14 @@ dev-create-db:
 dev-reset: dev-drop-db dev-create-db migrate  ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state.
 
 update-test-mmdb:  ## Update test GeoIP and ASN Databases
-	curl -L https://raw.githubusercontent.com/maxmind/MaxMind-DB/refs/heads/main/test-data/GeoLite2-ASN-Test.mmdb -o ${PWD}/tests/GeoLite2-ASN-Test.mmdb
+	curl \
-	curl -L https://raw.githubusercontent.com/maxmind/MaxMind-DB/refs/heads/main/test-data/GeoLite2-City-Test.mmdb -o ${PWD}/tests/GeoLite2-City-Test.mmdb
+		-L \
+		-o ${PWD}/tests/geoip/GeoLite2-ASN-Test.mmdb \
+		https://raw.githubusercontent.com/maxmind/MaxMind-DB/refs/heads/main/test-data/GeoLite2-ASN-Test.mmdb
+	curl \
+		-L \
+		-o ${PWD}/tests/geoip/GeoLite2-City-Test.mmdb \
+		https://raw.githubusercontent.com/maxmind/MaxMind-DB/refs/heads/main/test-data/GeoLite2-City-Test.mmdb
 
 bump:  ## Bump authentik version. Usage: make bump version=20xx.xx.xx
 ifndef version

authentik/root/test_plugin.py

@@ -6,7 +6,7 @@ import pytest
 from cryptography.hazmat.backends.openssl.backend import backend
 
 from authentik import authentik_full_version
-from tests.e2e.utils import get_local_ip
+from tests.decorators import get_local_ip
 
 IS_CI = "CI" in environ
 

authentik/root/test_runner.py

@@ -69,8 +69,8 @@ class PytestTestRunner(DiscoverRunner):  # pragma: no cover
 
         # Test-specific configuration
         test_config = {
-            "events.context_processors.geoip": "tests/GeoLite2-City-Test.mmdb",
+            "events.context_processors.geoip": "tests/geoip/GeoLite2-City-Test.mmdb",
-            "events.context_processors.asn": "tests/GeoLite2-ASN-Test.mmdb",
+            "events.context_processors.asn": "tests/geoip/GeoLite2-ASN-Test.mmdb",
             "blueprints_dir": "./blueprints",
             "outposts.container_image_base": f"ghcr.io/goauthentik/dev-%(type)s:{get_docker_tag()}",
             "tenants.enabled": False,

internal/outpost/flow/session.go

@@ -15,5 +15,14 @@ func (fe *FlowExecutor) Session() *jwt.Token {
 		return nil
 	}
 	t, _, _ := jwt.NewParser().ParseUnverified(sc.Value, &SessionCookieClaims{})
+	// During testing the session cookie value is not a JWT but rather just the session ID
+	// in which case we wrap that in a pseudo-JWT
+	if t == nil {
+		return &jwt.Token{
+			Claims: &SessionCookieClaims{
+				SessionID: sc.Value,
+			},
+		}
+	}
 	return t
 }

internal/outpost/ldap/ldap.go

@@ -149,6 +149,7 @@ func (ls *LDAPServer) handleWSSessionEnd(ctx context.Context, msg ak.Event) erro
 	ls.log.Info("Disconnecting session due to session end event")
 	conn, ok := ls.connections[mmsg.SessionID]
 	if !ok {
+		ls.log.Warn("Could not disconnect session, connection not found")
 		return nil
 	}
 	delete(ls.connections, mmsg.SessionID)

pyproject.toml

@@ -256,6 +256,7 @@ module = [
   "authentik.tenants.*",
   "guardian.*",
   "lifecycle.*",
+  "tests.*",
   "tests.e2e.*",
   "tests.integration.*",
   "tests.openid_conformance.*",

scripts/generate_config.py

@@ -27,8 +27,8 @@ def generate_local_config() -> dict[str, Any]:
         "cert_discovery_dir": "./certs",
         "events": {
             "processors": {
-                "geoip": "tests/GeoLite2-City-Test.mmdb",
+                "geoip": "tests/geoip/GeoLite2-City-Test.mmdb",
-                "asn": "tests/GeoLite2-ASN-Test.mmdb",
+                "asn": "tests/geoip/GeoLite2-ASN-Test.mmdb",
             }
         },
         "storage": {

tests/decorators.py

@@ -0,0 +1,77 @@
+import socket
+from collections.abc import Callable
+from functools import lru_cache, wraps
+from os import environ, getenv
+from typing import Any
+
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+from django.test.testcases import TransactionTestCase
+from selenium.common.exceptions import (
+    NoSuchElementException,
+    TimeoutException,
+    WebDriverException,
+)
+from structlog.stdlib import get_logger
+
+IS_CI = "CI" in environ
+RETRIES = int(environ.get("RETRIES", "3")) if IS_CI else 1
+SHADOW_ROOT_RETRIES = 5
+
+JSONType = dict[str, Any] | list[Any] | str | int | float | bool | None
+
+
+def get_local_ip(override=True) -> str:
+    """Get the local machine's IP"""
+    if (local_ip := getenv("LOCAL_IP")) and override:
+        return local_ip
+    hostname = socket.gethostname()
+    try:
+        return socket.gethostbyname(hostname)
+    except socket.gaierror:
+        return "0.0.0.0"
+
+
+@lru_cache
+def get_loader():
+    """Thin wrapper to lazily get a Migration Loader, only when it's needed
+    and only once"""
+    return MigrationLoader(connection)
+
+
+def retry(max_retires=RETRIES, exceptions=None):
+    """Retry test multiple times. Default to catching Selenium Timeout Exception"""
+
+    if not exceptions:
+        exceptions = [WebDriverException, TimeoutException, NoSuchElementException]
+
+    logger = get_logger()
+
+    def retry_actual(func: Callable):
+        """Retry test multiple times"""
+        count = 1
+
+        @wraps(func)
+        def wrapper(self: TransactionTestCase, *args, **kwargs):
+            """Run test again if we're below max_retries, including tearDown and
+            setUp. Otherwise raise the error"""
+            nonlocal count
+            try:
+                return func(self, *args, **kwargs)
+
+            except tuple(exceptions) as exc:
+                count += 1
+                if count > max_retires:
+                    logger.debug("Exceeded retry count", exc=exc, test=self)
+
+                    raise exc
+                logger.debug("Retrying on error", exc=exc, test=self)
+                self.tearDown()
+                self._post_teardown()
+                self._pre_setup()
+                self.setUp()
+                return wrapper(self, *args, **kwargs)
+
+        return wrapper
+
+    return retry_actual

tests/docker.py

@@ -1,6 +1,5 @@
 """authentik e2e testing utilities"""
 
-from os import environ
 from time import sleep
 from typing import Any
 from unittest.case import TestCase
@@ -13,8 +12,6 @@ from docker.models.networks import Network
 from authentik.lib.generators import generate_id
 from authentik.root.test_runner import get_docker_tag
 
-IS_CI = "CI" in environ
-
 
 class DockerTestCase(TestCase):
     """Mixin for dealing with containers"""

tests/e2e/test_endpoints_flow.py

@@ -8,7 +8,8 @@ from authentik.endpoints.models import Device, EndpointStage, StageMode
 from authentik.events.models import Event, EventAction
 from authentik.flows.models import Flow, FlowStageBinding
 from authentik.lib.generators import generate_id
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestEndpointsFlow(SeleniumTestCase):

tests/e2e/test_flows_authenticators.py

@@ -18,7 +18,8 @@ from authentik.stages.authenticator_static.models import (
     StaticToken,
 )
 from authentik.stages.authenticator_totp.models import AuthenticatorTOTPStage, TOTPDevice
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsAuthenticator(SeleniumTestCase):

tests/e2e/test_flows_authenticators_webauthn.py

@@ -15,8 +15,9 @@ from authentik.stages.authenticator_webauthn.models import (
     WebAuthnDevice,
 )
 from authentik.stages.identification.models import IdentificationStage
+from tests.decorators import retry
 from tests.e2e.test_flows_login_sfe import login_sfe
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsAuthenticatorWebAuthn(SeleniumTestCase):

tests/e2e/test_flows_enroll.py

@@ -12,7 +12,8 @@ from authentik.flows.models import Flow
 from authentik.lib.config import CONFIG
 from authentik.lib.generators import generate_id
 from authentik.stages.identification.models import IdentificationStage
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsEnroll(SeleniumTestCase):

tests/e2e/test_flows_login.py

@@ -2,7 +2,8 @@
 
 from authentik.blueprints.tests import apply_blueprint
 from authentik.flows.models import Flow
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsLogin(SeleniumTestCase):

tests/e2e/test_flows_login_sfe.py

@@ -10,7 +10,8 @@ from authentik.blueprints.tests import apply_blueprint
 from authentik.core.models import User
 from authentik.flows.models import NotConfiguredAction
 from authentik.stages.authenticator_validate.models import AuthenticatorValidateStage, DeviceClasses
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 def login_sfe(driver: WebDriver, user: User):

tests/e2e/test_flows_recovery.py

@@ -13,7 +13,8 @@ from authentik.flows.models import Flow
 from authentik.lib.config import CONFIG
 from authentik.lib.generators import generate_id
 from authentik.stages.identification.models import IdentificationStage
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsRecovery(SeleniumTestCase):

tests/e2e/test_flows_stage_setup.py

@@ -8,7 +8,8 @@ from authentik.core.models import User
 from authentik.flows.models import Flow, FlowDesignation
 from authentik.lib.generators import generate_key
 from authentik.stages.password.models import PasswordStage
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestFlowsStageSetup(SeleniumTestCase):

tests/e2e/test_provider_ldap.py

@@ -1,12 +1,13 @@
 """LDAP and Outpost e2e tests"""
 
 from dataclasses import asdict
+from time import sleep
 
 from ldap3 import ALL, ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES, SUBTREE, Connection, Server
-from ldap3.core.exceptions import LDAPInvalidCredentialsResult
+from ldap3.core.exceptions import LDAPInvalidCredentialsResult, LDAPSessionTerminatedByServerError
 
 from authentik.blueprints.tests import apply_blueprint, reconcile_app
-from authentik.core.models import Application, User
+from authentik.core.models import Application, AuthenticatedSession, User
 from authentik.core.tests.utils import create_test_user
 from authentik.events.models import Event, EventAction
 from authentik.flows.models import Flow
@@ -14,12 +15,32 @@ from authentik.lib.generators import generate_id
 from authentik.outposts.apps import MANAGED_OUTPOST
 from authentik.outposts.models import Outpost, OutpostConfig, OutpostType
 from authentik.providers.ldap.models import APIAccessMode, LDAPProvider
-from tests.e2e.utils import E2ETestCase, retry
+from tests.decorators import retry
+from tests.live import ChannelsE2ETestCase
 
 
-class TestProviderLDAP(E2ETestCase):
+def clean_response(response):
+    # Remove raw_attributes to make checking easier
+    for obj in response:
+        del obj["raw_attributes"]
+        del obj["raw_dn"]
+        obj["attributes"] = dict(obj["attributes"])
+        obj["attributes"].pop("uid", None)
+    return response
+
+
+class TestProviderLDAP(ChannelsE2ETestCase):
     """LDAP and Outpost e2e tests"""
 
+    def assert_list_dict_equal(self, expected: list[dict], actual: list[dict], match_key="dn"):
+        """Assert a list of dictionaries is identical, ignoring the ordering of items"""
+        self.assertEqual(len(expected), len(actual))
+        for res_item in actual:
+            all_matching = [x for x in expected if x[match_key] == res_item[match_key]]
+            self.assertEqual(len(all_matching), 1)
+            matching = all_matching[0]
+            self.assertDictEqual(res_item, matching)
+
     def start_ldap(self, outpost: Outpost):
         """Start ldap container based on outpost created"""
         self.run_container(
@@ -209,12 +230,7 @@ class TestProviderLDAP(E2ETestCase):
             search_scope=SUBTREE,
             attributes=[ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES],
         )
-        response: list = _connection.response
+        response = clean_response(_connection.response)
-        # Remove raw_attributes to make checking easier
-        for obj in response:
-            del obj["raw_attributes"]
-            del obj["raw_dn"]
-            obj["attributes"] = dict(obj["attributes"])
         o_user = outpost.user
         expected = [
             {
@@ -222,7 +238,6 @@ class TestProviderLDAP(E2ETestCase):
                 "attributes": {
                     "cn": o_user.username,
                     "sAMAccountName": o_user.username,
-                    "uid": o_user.uid,
                     "name": o_user.name,
                     "displayName": o_user.name,
                     "sn": o_user.name,
@@ -253,7 +268,6 @@ class TestProviderLDAP(E2ETestCase):
                 "attributes": {
                     "cn": embedded_account.username,
                     "sAMAccountName": embedded_account.username,
-                    "uid": embedded_account.uid,
                     "name": embedded_account.name,
                     "displayName": embedded_account.name,
                     "sn": embedded_account.name,
@@ -284,7 +298,6 @@ class TestProviderLDAP(E2ETestCase):
                 "attributes": {
                     "cn": self.user.username,
                     "sAMAccountName": self.user.username,
-                    "uid": self.user.uid,
                     "name": self.user.name,
                     "displayName": self.user.name,
                     "sn": self.user.name,
@@ -353,19 +366,13 @@ class TestProviderLDAP(E2ETestCase):
             search_scope=SUBTREE,
             attributes=[ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES],
         )
-        response: list = _connection.response
+        response = clean_response(_connection.response)
-        # Remove raw_attributes to make checking easier
-        for obj in response:
-            del obj["raw_attributes"]
-            del obj["raw_dn"]
-            obj["attributes"] = dict(obj["attributes"])
         expected = [
             {
                 "dn": f"cn={user.username},ou=users,dc=ldap,dc=goauthentik,dc=io",
                 "attributes": {
                     "cn": user.username,
                     "sAMAccountName": user.username,
-                    "uid": user.uid,
                     "name": user.name,
                     "displayName": user.name,
                     "sn": user.name,
@@ -397,15 +404,6 @@ class TestProviderLDAP(E2ETestCase):
         ]
         self.assert_list_dict_equal(expected, response)
 
-    def assert_list_dict_equal(self, expected: list[dict], actual: list[dict], match_key="dn"):
-        """Assert a list of dictionaries is identical, ignoring the ordering of items"""
-        self.assertEqual(len(expected), len(actual))
-        for res_item in actual:
-            all_matching = [x for x in expected if x[match_key] == res_item[match_key]]
-            self.assertEqual(len(all_matching), 1)
-            matching = all_matching[0]
-            self.assertDictEqual(res_item, matching)
-
     @retry()
     @apply_blueprint(
         "default/flow-default-authentication-flow.yaml",
@@ -469,11 +467,8 @@ class TestProviderLDAP(E2ETestCase):
             search_scope=SUBTREE,
             attributes=["cn"],
         )
-        response: list = _connection.response
+        response = clean_response(_connection.response)
-        # Remove raw_attributes to make checking easier
+
-        for obj in response:
-            del obj["raw_attributes"]
-            del obj["raw_dn"]
         o_user = outpost.user
         self.assert_list_dict_equal(
             [
@@ -501,3 +496,44 @@ class TestProviderLDAP(E2ETestCase):
             ],
             response,
         )
+
+    @retry()
+    @apply_blueprint(
+        "default/flow-default-authentication-flow.yaml",
+        "default/flow-default-invalidation-flow.yaml",
+    )
+    @reconcile_app("authentik_tenants")
+    @reconcile_app("authentik_outposts")
+    def test_ldap_bind_logout_search(self):
+        """Test bind + session deletion -> failed search"""
+        self._prepare()
+        server = Server("ldap://localhost:3389", get_info=ALL)
+        _connection = Connection(
+            server,
+            raise_exceptions=True,
+            user=f"cn={self.user.username},ou=users,dc=ldap,dc=goauthentik,dc=io",
+            password=self.user.username,
+        )
+        _connection.bind()
+        self.assertTrue(
+            Event.objects.filter(
+                action=EventAction.LOGIN,
+                user={
+                    "pk": self.user.pk,
+                    "email": self.user.email,
+                    "username": self.user.username,
+                },
+            )
+        )
+        c, _ = AuthenticatedSession.objects.filter(user_id=self.user.pk).delete()
+        self.assertGreaterEqual(c, 1)
+        # Give the sign out signal time to propagate
+        sleep(3)
+
+        with self.assertRaises(LDAPSessionTerminatedByServerError):
+            _connection.search(
+                "ou=Users,DC=ldaP,dc=goauthentik,dc=io",
+                "(objectClass=user)",
+                search_scope=SUBTREE,
+                attributes=[ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES],
+            )

tests/e2e/test_provider_oauth2_github.py

@@ -18,7 +18,8 @@ from authentik.providers.oauth2.models import (
     RedirectURI,
     RedirectURIMatchingMode,
 )
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderOAuth2Github(SeleniumTestCase):

tests/e2e/test_provider_oauth2_grafana.py

@@ -26,7 +26,8 @@ from authentik.providers.oauth2.models import (
     RedirectURIMatchingMode,
     ScopeMapping,
 )
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderOAuth2OAuth(SeleniumTestCase):

tests/e2e/test_provider_oidc.py

@@ -26,7 +26,8 @@ from authentik.providers.oauth2.models import (
     RedirectURIMatchingMode,
     ScopeMapping,
 )
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderOAuth2OIDC(SeleniumTestCase):

tests/e2e/test_provider_oidc_implicit.py

@@ -26,7 +26,8 @@ from authentik.providers.oauth2.models import (
     RedirectURIMatchingMode,
     ScopeMapping,
 )
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderOAuth2OIDCImplicit(SeleniumTestCase):

tests/e2e/test_provider_proxy.py

@@ -3,11 +3,8 @@
 from base64 import b64encode
 from dataclasses import asdict
 from json import dumps
-from sys import platform
 from time import sleep
-from unittest.case import skip, skipUnless
 
-from channels.testing import ChannelsLiveServerTestCase
 from jwt import decode
 from selenium.webdriver.common.by import By
 
@@ -15,10 +12,11 @@ from authentik.blueprints.tests import apply_blueprint, reconcile_app
 from authentik.core.models import Application
 from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
-from authentik.outposts.models import DockerServiceConnection, Outpost, OutpostConfig, OutpostType
+from authentik.outposts.models import Outpost, OutpostConfig, OutpostType
-from authentik.outposts.tasks import outpost_connection_discovery
 from authentik.providers.proxy.models import ProxyProvider
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.live import ChannelsE2ETestCase
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderProxy(SeleniumTestCase):
@@ -215,10 +213,7 @@ class TestProviderProxy(SeleniumTestCase):
         )
 
 
-# TODO: Fix flaky test
+class TestProviderProxyConnect(ChannelsE2ETestCase):
-@skip("Flaky test")
-@skipUnless(platform.startswith("linux"), "requires local docker")
-class TestProviderProxyConnect(ChannelsLiveServerTestCase):
     """Test Proxy connectivity over websockets"""
 
     @retry(exceptions=[AssertionError])
@@ -232,7 +227,6 @@ class TestProviderProxyConnect(ChannelsLiveServerTestCase):
     @reconcile_app("authentik_crypto")
     def test_proxy_connectivity(self):
         """Test proxy connectivity over websocket"""
-        outpost_connection_discovery()
         proxy: ProxyProvider = ProxyProvider.objects.create(
             name=generate_id(),
             authorization_flow=Flow.objects.get(
@@ -246,16 +240,25 @@ class TestProviderProxyConnect(ChannelsLiveServerTestCase):
         proxy.save()
         # we need to create an application to actually access the proxy
         Application.objects.create(name=generate_id(), slug=generate_id(), provider=proxy)
-        service_connection = DockerServiceConnection.objects.get(local=True)
+
         outpost: Outpost = Outpost.objects.create(
             name=generate_id(),
             type=OutpostType.PROXY,
-            service_connection=service_connection,
             _config=asdict(OutpostConfig(authentik_host=self.live_server_url, log_level="debug")),
         )
         outpost.providers.add(proxy)
         outpost.build_user_permissions(outpost.user)
 
+        self.run_container(
+            image=self.get_container_image("ghcr.io/goauthentik/dev-proxy"),
+            ports={
+                "9000": "9000",
+            },
+            environment={
+                "AUTHENTIK_TOKEN": outpost.token.key,
+            },
+        )
+
         # Wait until outpost healthcheck succeeds
         healthcheck_retries = 0
         while healthcheck_retries < 50:  # noqa: PLR2004

tests/e2e/test_provider_proxy_forward.py

@@ -13,7 +13,8 @@ from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
 from authentik.outposts.models import Outpost, OutpostType
 from authentik.providers.proxy.models import ProxyMode, ProxyProvider
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderProxyForward(SeleniumTestCase):

tests/e2e/test_provider_rac.py

@@ -11,7 +11,8 @@ from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
 from authentik.outposts.models import Outpost, OutpostType
 from authentik.providers.rac.models import Endpoint, Protocols, RACProvider
-from tests.e2e.utils import ChannelsSeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import ChannelsSeleniumTestCase
 
 
 class TestProviderRAC(ChannelsSeleniumTestCase):

tests/e2e/test_provider_radius.py

@@ -13,7 +13,8 @@ from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id, generate_key
 from authentik.outposts.models import Outpost, OutpostConfig, OutpostType
 from authentik.providers.radius.models import RadiusProvider
-from tests.e2e.utils import E2ETestCase, retry
+from tests.decorators import retry
+from tests.live import E2ETestCase
 
 
 class TestProviderRadius(E2ETestCase):

tests/e2e/test_provider_saml.py

@@ -15,7 +15,8 @@ from authentik.lib.generators import generate_id
 from authentik.policies.expression.models import ExpressionPolicy
 from authentik.policies.models import PolicyBinding
 from authentik.providers.saml.models import SAMLBindings, SAMLPropertyMapping, SAMLProvider
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderSAML(SeleniumTestCase):

tests/e2e/test_provider_ws_fed.py

@@ -12,7 +12,8 @@ from authentik.enterprise.providers.ws_federation.models import WSFederationProv
 from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
 from authentik.providers.saml.models import SAMLPropertyMapping
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class TestProviderWSFed(SeleniumTestCase):

tests/e2e/test_source_ldap_samba.py

@@ -12,7 +12,8 @@ from authentik.sources.ldap.sync.groups import GroupLDAPSynchronizer
 from authentik.sources.ldap.sync.membership import MembershipLDAPSynchronizer
 from authentik.sources.ldap.sync.users import UserLDAPSynchronizer
 from authentik.tasks.models import Task
-from tests.e2e.utils import E2ETestCase, retry
+from tests.decorators import retry
+from tests.live import E2ETestCase
 
 
 class TestSourceLDAPSamba(E2ETestCase):

tests/e2e/test_source_oauth_oauth1.py

@@ -16,7 +16,8 @@ from authentik.sources.oauth.models import OAuthSource
 from authentik.sources.oauth.types.registry import SourceType, registry
 from authentik.sources.oauth.views.callback import OAuthCallback
 from authentik.stages.identification.models import IdentificationStage
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 
 class OAuth1Callback(OAuthCallback):

tests/e2e/test_source_oauth_oauth2.py

@@ -4,6 +4,10 @@ from pathlib import Path
 from time import sleep
 
 from docker.types import Healthcheck
+from selenium.common.exceptions import (
+    NoSuchElementException,
+    TimeoutException,
+)
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions as ec
@@ -15,7 +19,8 @@ from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
 from authentik.sources.oauth.models import OAuthSource
 from authentik.stages.identification.models import IdentificationStage
-from tests.e2e.utils import NoSuchElementException, SeleniumTestCase, TimeoutException, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 MAX_REFRESH_RETRIES = 5
 INTERFACE_TIMEOUT = 10

tests/e2e/test_source_saml.py

@@ -16,7 +16,8 @@ from authentik.flows.models import Flow
 from authentik.lib.generators import generate_id
 from authentik.sources.saml.models import SAMLBindingTypes, SAMLSource
 from authentik.stages.identification.models import IdentificationStage
-from tests.e2e.utils import SeleniumTestCase, retry
+from tests.decorators import retry
+from tests.selenium import SeleniumTestCase
 
 IDP_CERT = """-----BEGIN CERTIFICATE-----
 MIIDXTCCAkWgAwIBAgIJALmVVuDWu4NYMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV

tests/e2e/test_source_scim.py

@@ -8,7 +8,8 @@ from docker.types import Healthcheck
 from authentik.lib.generators import generate_id
 from authentik.lib.utils.http import get_http_session
 from authentik.sources.scim.models import SCIMSource
-from tests.e2e.utils import E2ETestCase, retry
+from tests.decorators import retry
+from tests.live import E2ETestCase
 
 TEST_POLL_MAX = 25
 

tests/live.py

@@ -0,0 +1,54 @@
+from sys import stderr
+
+from channels.testing import ChannelsLiveServerTestCase
+from django.apps import apps
+from django.contrib.staticfiles.testing import StaticLiveServerTestCase
+from dramatiq import get_broker
+from structlog.stdlib import get_logger
+
+from authentik.core.models import User
+from authentik.core.tests.utils import create_test_admin_user
+from authentik.tasks.test import use_test_broker
+from tests._process import TestDatabaseProcess
+from tests.decorators import IS_CI, get_local_ip
+from tests.docker import DockerTestCase
+
+
+class E2ETestMixin(DockerTestCase):
+    host = get_local_ip()
+    user: User
+    serve_static = True
+    ProtocolServerProcess = TestDatabaseProcess
+
+    def setUp(self):
+        if IS_CI:
+            print("::group::authentik Logs", file=stderr)
+        apps.get_app_config("authentik_tenants").ready()
+        self.wait_timeout = 60
+        self.logger = get_logger()
+        self.user = create_test_admin_user()
+        super().setUp()
+
+    @classmethod
+    def _pre_setup(cls):
+        use_test_broker()
+        return super()._pre_setup()
+
+    def _post_teardown(self):
+        broker = get_broker()
+        broker.flush_all()
+        broker.close()
+        return super()._post_teardown()
+
+    def tearDown(self):
+        if IS_CI:
+            print("::endgroup::", file=stderr)
+        super().tearDown()
+
+
+class E2ETestCase(E2ETestMixin, StaticLiveServerTestCase):
+    """E2E Test case with django static live server"""
+
+
+class ChannelsE2ETestCase(E2ETestMixin, ChannelsLiveServerTestCase):
+    """E2E Test case with channels live server (websocket + static)"""

tests/openid_conformance/base.py

@@ -7,8 +7,8 @@ from selenium.webdriver.support import expected_conditions as ec
 
 from authentik.blueprints.tests import apply_blueprint, reconcile_app
 from authentik.providers.oauth2.models import OAuth2Provider
-from tests.e2e.utils import SeleniumTestCase
 from tests.openid_conformance.conformance import Conformance
+from tests.selenium import SeleniumTestCase
 
 
 class TestOpenIDConformance(SeleniumTestCase):

tests/openid_conformance/test_basic.py

@@ -1,4 +1,4 @@
-from tests.e2e.utils import retry
+from tests.decorators import retry
 from tests.openid_conformance.base import TestOpenIDConformance
 
 

tests/openid_conformance/test_implicit.py

@@ -1,4 +1,4 @@
-from tests.e2e.utils import retry
+from tests.decorators import retry
 from tests.openid_conformance.base import TestOpenIDConformance
 
 

tests/selenium.py

@@ -1,26 +1,15 @@
 """authentik e2e testing utilities"""
 
-import socket
+from functools import cached_property
-from collections.abc import Callable
-from functools import cached_property, lru_cache, wraps
 from json import JSONDecodeError, dumps, loads
-from os import environ, getenv
 from pathlib import Path
-from sys import stderr
 from tempfile import gettempdir
 from time import sleep
-from typing import Any
 from urllib.parse import urlencode
 
-from channels.testing import ChannelsLiveServerTestCase
-from django.apps import apps
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
-from django.db import connection
-from django.db.migrations.loader import MigrationLoader
-from django.test.testcases import TransactionTestCase
 from django.urls import reverse
 from docker.models.containers import Container
-from dramatiq import get_broker
 from requests import RequestException
 from selenium import webdriver
 from selenium.common.exceptions import (
@@ -38,72 +27,12 @@ from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.remote.webelement import WebElement
 from selenium.webdriver.support import expected_conditions as ec
 from selenium.webdriver.support.wait import WebDriverWait
-from structlog.stdlib import get_logger
 
 from authentik.core.api.users import UserSerializer
 from authentik.core.models import User
-from authentik.core.tests.utils import create_test_admin_user
 from authentik.lib.utils.http import get_http_session
-from authentik.tasks.test import use_test_broker
+from tests.decorators import IS_CI, RETRIES, SHADOW_ROOT_RETRIES, JSONType
-from tests.docker import DockerTestCase
+from tests.live import ChannelsE2ETestCase, E2ETestMixin
-from tests.e2e._process import TestDatabaseProcess
-
-IS_CI = "CI" in environ
-RETRIES = int(environ.get("RETRIES", "3")) if IS_CI else 1
-SHADOW_ROOT_RETRIES = 5
-
-JSONType = dict[str, Any] | list[Any] | str | int | float | bool | None
-
-
-def get_local_ip(override=True) -> str:
-    """Get the local machine's IP"""
-    if (local_ip := getenv("LOCAL_IP")) and override:
-        return local_ip
-    hostname = socket.gethostname()
-    try:
-        return socket.gethostbyname(hostname)
-    except socket.gaierror:
-        return "0.0.0.0"
-
-
-class E2ETestMixin(DockerTestCase):
-    host = get_local_ip()
-    user: User
-    serve_static = True
-    ProtocolServerProcess = TestDatabaseProcess
-
-    def setUp(self):
-        if IS_CI:
-            print("::group::authentik Logs", file=stderr)
-        apps.get_app_config("authentik_tenants").ready()
-        self.wait_timeout = 60
-        self.logger = get_logger()
-        self.user = create_test_admin_user()
-        super().setUp()
-
-    @classmethod
-    def _pre_setup(cls):
-        use_test_broker()
-        return super()._pre_setup()
-
-    def _post_teardown(self):
-        broker = get_broker()
-        broker.flush_all()
-        broker.close()
-        return super()._post_teardown()
-
-    def tearDown(self):
-        if IS_CI:
-            print("::endgroup::", file=stderr)
-        super().tearDown()
-
-
-class E2ETestCase(E2ETestMixin, StaticLiveServerTestCase):
-    """E2E Test case with django static live server"""
-
-
-class ChannelsE2ETestCase(E2ETestMixin, ChannelsLiveServerTestCase):
-    """E2E Test case with channels live server (websocket + static)"""
 
 
 class SeleniumTestMixin(E2ETestMixin):
@@ -454,48 +383,3 @@ class SeleniumTestCase(SeleniumTestMixin, StaticLiveServerTestCase):
 
 class ChannelsSeleniumTestCase(SeleniumTestMixin, ChannelsE2ETestCase):
     """Selenium Test case with channels live server (websocket + static)"""
-
-
-@lru_cache
-def get_loader():
-    """Thin wrapper to lazily get a Migration Loader, only when it's needed
-    and only once"""
-    return MigrationLoader(connection)
-
-
-def retry(max_retires=RETRIES, exceptions=None):
-    """Retry test multiple times. Default to catching Selenium Timeout Exception"""
-
-    if not exceptions:
-        exceptions = [WebDriverException, TimeoutException, NoSuchElementException]
-
-    logger = get_logger()
-
-    def retry_actual(func: Callable):
-        """Retry test multiple times"""
-        count = 1
-
-        @wraps(func)
-        def wrapper(self: TransactionTestCase, *args, **kwargs):
-            """Run test again if we're below max_retries, including tearDown and
-            setUp. Otherwise raise the error"""
-            nonlocal count
-            try:
-                return func(self, *args, **kwargs)
-
-            except tuple(exceptions) as exc:
-                count += 1
-                if count > max_retires:
-                    logger.debug("Exceeded retry count", exc=exc, test=self)
-
-                    raise exc
-                logger.debug("Retrying on error", exc=exc, test=self)
-                self.tearDown()
-                self._post_teardown()
-                self._pre_setup()
-                self.setUp()
-                return wrapper(self, *args, **kwargs)
-
-        return wrapper
-
-    return retry_actual