e2e: add @retry decorator to make e2e tests more reliable
This commit is contained in:
parent
ef021495ef
commit
aeee3ad7f9
|
@ -8,7 +8,7 @@ from docker.types import Healthcheck
|
|||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support import expected_conditions as ec
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
|
||||
from passbook.stages.email.models import EmailStage, EmailTemplates
|
||||
from passbook.stages.identification.models import IdentificationStage
|
||||
|
@ -34,6 +34,7 @@ class TestFlowsEnroll(SeleniumTestCase):
|
|||
),
|
||||
}
|
||||
|
||||
@retry()
|
||||
def test_enroll_2_step(self):
|
||||
"""Test 2-step enroll flow"""
|
||||
# First stage fields
|
||||
|
@ -119,6 +120,7 @@ class TestFlowsEnroll(SeleniumTestCase):
|
|||
"foo@bar.baz",
|
||||
)
|
||||
|
||||
@retry()
|
||||
@override_settings(EMAIL_BACKEND="django.core.mail.backends.smtp.EmailBackend")
|
||||
def test_enroll_email(self):
|
||||
"""Test enroll with Email verification"""
|
||||
|
|
|
@ -5,13 +5,14 @@ from unittest.case import skipUnless
|
|||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
|
||||
|
||||
@skipUnless(platform.startswith("linux"), "requires local docker")
|
||||
class TestFlowsLogin(SeleniumTestCase):
|
||||
"""test default login flow"""
|
||||
|
||||
@retry()
|
||||
def test_login(self):
|
||||
"""test default login flow"""
|
||||
self.driver.get(f"{self.live_server_url}/flows/default-authentication-flow/")
|
||||
|
|
|
@ -12,7 +12,7 @@ from selenium.webdriver.common.by import By
|
|||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.webdriver.support import expected_conditions as ec
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.flows.models import Flow, FlowStageBinding
|
||||
from passbook.stages.otp_validate.models import OTPValidateStage
|
||||
|
||||
|
@ -21,6 +21,7 @@ from passbook.stages.otp_validate.models import OTPValidateStage
|
|||
class TestFlowsOTP(SeleniumTestCase):
|
||||
"""test flow with otp stages"""
|
||||
|
||||
@retry()
|
||||
def test_otp_validate(self):
|
||||
"""test flow with otp stages"""
|
||||
sleep(1)
|
||||
|
@ -52,6 +53,7 @@ class TestFlowsOTP(SeleniumTestCase):
|
|||
USER().username,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_otp_totp_setup(self):
|
||||
"""test TOTP Setup stage"""
|
||||
flow: Flow = Flow.objects.get(slug="default-authentication-flow")
|
||||
|
@ -98,6 +100,7 @@ class TestFlowsOTP(SeleniumTestCase):
|
|||
|
||||
self.assertTrue(TOTPDevice.objects.filter(user=USER(), confirmed=True).exists())
|
||||
|
||||
@retry()
|
||||
def test_otp_static_setup(self):
|
||||
"""test Static OTP Setup stage"""
|
||||
flow: Flow = Flow.objects.get(slug="default-authentication-flow")
|
||||
|
|
|
@ -5,7 +5,7 @@ from unittest.case import skipUnless
|
|||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.core.models import User
|
||||
from passbook.flows.models import Flow, FlowDesignation
|
||||
from passbook.providers.oauth2.generators import generate_client_secret
|
||||
|
@ -16,6 +16,7 @@ from passbook.stages.password.models import PasswordStage
|
|||
class TestFlowsStageSetup(SeleniumTestCase):
|
||||
"""test stage setup flows"""
|
||||
|
||||
@retry()
|
||||
def test_password_change(self):
|
||||
"""test password change flow"""
|
||||
# Ensure that password stage has change_flow set
|
||||
|
|
|
@ -9,7 +9,7 @@ from selenium.webdriver.common.by import By
|
|||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.webdriver.support import expected_conditions as ec
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.core.models import Application
|
||||
from passbook.flows.models import Flow
|
||||
from passbook.policies.expression.models import ExpressionPolicy
|
||||
|
@ -61,6 +61,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
|
|||
},
|
||||
}
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_implied(self):
|
||||
"""test OAuth Provider flow (default authorization flow with implied consent)"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -115,6 +116,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
|
|||
USER().username,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_explicit(self):
|
||||
"""test OAuth Provider flow (default authorization flow with explicit consent)"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -184,6 +186,7 @@ class TestProviderOAuth2Github(SeleniumTestCase):
|
|||
USER().username,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_denied(self):
|
||||
"""test OAuth Provider flow (default authorization flow, denied)"""
|
||||
# Bootstrap all needed objects
|
||||
|
|
|
@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys
|
|||
from selenium.webdriver.support import expected_conditions as ec
|
||||
from structlog import get_logger
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.core.models import Application
|
||||
from passbook.crypto.models import CertificateKeyPair
|
||||
from passbook.flows.models import Flow
|
||||
|
@ -80,6 +80,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
|
|||
},
|
||||
}
|
||||
|
||||
@retry()
|
||||
def test_redirect_uri_error(self):
|
||||
"""test OpenID Provider flow (invalid redirect URI, check error message)"""
|
||||
sleep(1)
|
||||
|
@ -122,6 +123,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
|
|||
"Redirect URI Error",
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_implied(self):
|
||||
"""test OpenID Provider flow (default authorization flow with implied consent)"""
|
||||
sleep(1)
|
||||
|
@ -183,6 +185,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
|
|||
USER().email,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_authorization_logout(self):
|
||||
"""test OpenID Provider flow with logout"""
|
||||
sleep(1)
|
||||
|
@ -252,6 +255,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
|
|||
)
|
||||
self.driver.find_element(By.ID, "logout").click()
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_explicit(self):
|
||||
"""test OpenID Provider flow (default authorization flow with explicit consent)"""
|
||||
sleep(1)
|
||||
|
@ -325,6 +329,7 @@ class TestProviderOAuth2OAuth(SeleniumTestCase):
|
|||
USER().email,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_authorization_denied(self):
|
||||
"""test OpenID Provider flow (default authorization with access deny)"""
|
||||
sleep(1)
|
||||
|
|
|
@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys
|
|||
from selenium.webdriver.support import expected_conditions as ec
|
||||
from structlog import get_logger
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.core.models import Application
|
||||
from passbook.crypto.models import CertificateKeyPair
|
||||
from passbook.flows.models import Flow
|
||||
|
@ -76,6 +76,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
|
|||
LOGGER.info("Container failed healthcheck")
|
||||
sleep(1)
|
||||
|
||||
@retry()
|
||||
def test_redirect_uri_error(self):
|
||||
"""test OpenID Provider flow (invalid redirect URI, check error message)"""
|
||||
sleep(1)
|
||||
|
@ -119,6 +120,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
|
|||
"Redirect URI Error",
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_implied(self):
|
||||
"""test OpenID Provider flow (default authorization flow with implied consent)"""
|
||||
sleep(1)
|
||||
|
@ -169,6 +171,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
|
|||
self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
|
||||
self.assertEqual(body["UserInfo"]["email"], USER().email)
|
||||
|
||||
@retry()
|
||||
def test_authorization_consent_explicit(self):
|
||||
"""test OpenID Provider flow (default authorization flow with explicit consent)"""
|
||||
sleep(1)
|
||||
|
@ -229,6 +232,7 @@ class TestProviderOAuth2OIDC(SeleniumTestCase):
|
|||
self.assertEqual(body["IDTokenClaims"]["email"], USER().email)
|
||||
self.assertEqual(body["UserInfo"]["email"], USER().email)
|
||||
|
||||
@retry()
|
||||
def test_authorization_denied(self):
|
||||
"""test OpenID Provider flow (default authorization with access deny)"""
|
||||
sleep(1)
|
||||
|
|
|
@ -11,7 +11,7 @@ from docker.models.containers import Container
|
|||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook import __version__
|
||||
from passbook.core.models import Application
|
||||
from passbook.flows.models import Flow
|
||||
|
@ -57,6 +57,7 @@ class TestProviderProxy(SeleniumTestCase):
|
|||
)
|
||||
return container
|
||||
|
||||
@retry()
|
||||
def test_proxy_simple(self):
|
||||
"""Test simple outpost setup with single provider"""
|
||||
proxy: ProxyProvider = ProxyProvider.objects.create(
|
||||
|
@ -110,6 +111,7 @@ class TestProviderProxy(SeleniumTestCase):
|
|||
class TestProviderProxyConnect(ChannelsLiveServerTestCase):
|
||||
"""Test Proxy connectivity over websockets"""
|
||||
|
||||
@retry()
|
||||
def test_proxy_connectivity(self):
|
||||
"""Test proxy connectivity over websocket"""
|
||||
SeleniumTestCase().apply_default_data()
|
||||
|
|
|
@ -12,7 +12,7 @@ from selenium.webdriver.common.keys import Keys
|
|||
from selenium.webdriver.support import expected_conditions as ec
|
||||
from structlog import get_logger
|
||||
|
||||
from e2e.utils import USER, SeleniumTestCase
|
||||
from e2e.utils import USER, SeleniumTestCase, retry
|
||||
from passbook.core.models import Application
|
||||
from passbook.crypto.models import CertificateKeyPair
|
||||
from passbook.flows.models import Flow
|
||||
|
@ -66,6 +66,7 @@ class TestProviderSAML(SeleniumTestCase):
|
|||
LOGGER.info("Container failed healthcheck")
|
||||
sleep(1)
|
||||
|
||||
@retry()
|
||||
def test_sp_initiated_implicit(self):
|
||||
"""test SAML Provider flow SP-initiated flow (implicit consent)"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -105,6 +106,7 @@ class TestProviderSAML(SeleniumTestCase):
|
|||
self.assertEqual(body["attr"]["mail"], [USER().email])
|
||||
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
|
||||
|
||||
@retry()
|
||||
def test_sp_initiated_explicit(self):
|
||||
"""test SAML Provider flow SP-initiated flow (explicit consent)"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -150,6 +152,7 @@ class TestProviderSAML(SeleniumTestCase):
|
|||
self.assertEqual(body["attr"]["mail"], [USER().email])
|
||||
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
|
||||
|
||||
@retry()
|
||||
def test_idp_initiated_implicit(self):
|
||||
"""test SAML Provider flow IdP-initiated flow (implicit consent)"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -195,6 +198,7 @@ class TestProviderSAML(SeleniumTestCase):
|
|||
self.assertEqual(body["attr"]["mail"], [USER().email])
|
||||
self.assertEqual(body["attr"]["uid"], [str(USER().pk)])
|
||||
|
||||
@retry()
|
||||
def test_sp_initiated_denied(self):
|
||||
"""test SAML Provider flow SP-initiated flow (Policy denies access)"""
|
||||
# Bootstrap all needed objects
|
||||
|
|
|
@ -14,7 +14,7 @@ from selenium.webdriver.support import expected_conditions as ec
|
|||
from structlog import get_logger
|
||||
from yaml import safe_dump
|
||||
|
||||
from e2e.utils import SeleniumTestCase
|
||||
from e2e.utils import SeleniumTestCase, retry
|
||||
from passbook.flows.models import Flow
|
||||
from passbook.providers.oauth2.generators import (
|
||||
generate_client_id,
|
||||
|
@ -106,6 +106,7 @@ class TestSourceOAuth2(SeleniumTestCase):
|
|||
consumer_secret=self.client_secret,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_oauth_enroll(self):
|
||||
"""test OAuth Source With With OIDC"""
|
||||
self.create_objects()
|
||||
|
@ -159,6 +160,7 @@ class TestSourceOAuth2(SeleniumTestCase):
|
|||
"admin@example.com",
|
||||
)
|
||||
|
||||
@retry()
|
||||
@override_settings(SESSION_COOKIE_SAMESITE="strict")
|
||||
def test_oauth_samesite_strict(self):
|
||||
"""test OAuth Source With SameSite set to strict
|
||||
|
@ -195,6 +197,7 @@ class TestSourceOAuth2(SeleniumTestCase):
|
|||
"Authentication Failed.",
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_oauth_enroll_auth(self):
|
||||
"""test OAuth Source With With OIDC (enroll and authenticate again)"""
|
||||
self.test_oauth_enroll()
|
||||
|
@ -291,6 +294,7 @@ class TestSourceOAuth1(SeleniumTestCase):
|
|||
consumer_secret=self.client_secret,
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_oauth_enroll(self):
|
||||
"""test OAuth Source With With OIDC"""
|
||||
self.create_objects()
|
||||
|
|
|
@ -10,7 +10,7 @@ from selenium.webdriver.common.keys import Keys
|
|||
from selenium.webdriver.support import expected_conditions as ec
|
||||
from structlog import get_logger
|
||||
|
||||
from e2e.utils import SeleniumTestCase
|
||||
from e2e.utils import SeleniumTestCase, retry
|
||||
from passbook.crypto.models import CertificateKeyPair
|
||||
from passbook.flows.models import Flow
|
||||
from passbook.sources.saml.models import SAMLBindingTypes, SAMLSource
|
||||
|
@ -92,6 +92,7 @@ class TestSourceSAML(SeleniumTestCase):
|
|||
},
|
||||
}
|
||||
|
||||
@retry()
|
||||
def test_idp_redirect(self):
|
||||
"""test SAML Source With redirect binding"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -141,6 +142,7 @@ class TestSourceSAML(SeleniumTestCase):
|
|||
self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_idp_post(self):
|
||||
"""test SAML Source With post binding"""
|
||||
# Bootstrap all needed objects
|
||||
|
@ -192,6 +194,7 @@ class TestSourceSAML(SeleniumTestCase):
|
|||
self.driver.find_element(By.ID, "id_username").get_attribute("value"), ""
|
||||
)
|
||||
|
||||
@retry()
|
||||
def test_idp_post_auto(self):
|
||||
"""test SAML Source With post binding (auto redirect)"""
|
||||
# Bootstrap all needed objects
|
||||
|
|
37
e2e/utils.py
37
e2e/utils.py
|
@ -1,19 +1,22 @@
|
|||
"""passbook e2e testing utilities"""
|
||||
from functools import wraps
|
||||
from glob import glob
|
||||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from inspect import getmembers, isfunction
|
||||
from os import environ, makedirs
|
||||
from time import sleep, time
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from django.apps import apps
|
||||
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
|
||||
from django.db import connection, transaction
|
||||
from django.db.utils import IntegrityError
|
||||
from django.shortcuts import reverse
|
||||
from django.test.testcases import TestCase
|
||||
from docker import DockerClient, from_env
|
||||
from docker.models.containers import Container
|
||||
from selenium import webdriver
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
|
||||
from selenium.webdriver.remote.webdriver import WebDriver
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
@ -123,3 +126,35 @@ class SeleniumTestCase(StaticLiveServerTestCase):
|
|||
func(apps, schema_editor)
|
||||
except IntegrityError:
|
||||
pass
|
||||
|
||||
|
||||
def retry(max_retires=3, exceptions=None):
|
||||
"""Retry test multiple times. Default to catching Selenium Timeout Exception"""
|
||||
|
||||
if not exceptions:
|
||||
exceptions = [TimeoutException]
|
||||
|
||||
def retry_actual(func: Callable):
|
||||
"""Retry test multiple times"""
|
||||
count = 1
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(self: TestCase, *args, **kwargs):
|
||||
"""Run test again if we're below max_retries, including tearDown and
|
||||
setUp. Otherwise raise the error"""
|
||||
nonlocal count
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
# pylint: disable=catching-non-exception
|
||||
except tuple(exceptions) as exc:
|
||||
count += 1
|
||||
if count > max_retires:
|
||||
# pylint: disable=raising-non-exception
|
||||
raise exc
|
||||
self.tearDown()
|
||||
self.setUp()
|
||||
return wrapper(self, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return retry_actual
|
||||
|
|
|
@ -6348,7 +6348,7 @@ definitions:
|
|||
for input-based policies.
|
||||
type: boolean
|
||||
re_evaluate_policies:
|
||||
title: Evaluate on call
|
||||
title: Re evaluate policies
|
||||
description: Evaluate policies when the Stage is present to the user.
|
||||
type: boolean
|
||||
order:
|
||||
|
|
Reference in New Issue