diff --git a/authentik/outposts/controllers/k8s/base.py b/authentik/outposts/controllers/k8s/base.py
index d9f8a3c98..44506b5d8 100644
--- a/authentik/outposts/controllers/k8s/base.py
+++ b/authentik/outposts/controllers/k8s/base.py
@@ -20,6 +20,11 @@ if TYPE_CHECKING:
 T = TypeVar("T", V1Pod, V1Deployment)
 
 
+def get_version() -> str:
+    """Wrapper for __version__ to make testing easier"""
+    return __version__
+
+
 class KubernetesObjectReconciler(Generic[T]):
     """Base Kubernetes Reconciler, handles the basic logic."""
 
@@ -146,13 +151,13 @@ class KubernetesObjectReconciler(Generic[T]):
         return V1ObjectMeta(
             namespace=self.namespace,
             labels={
-                "app.kubernetes.io/name": f"authentik-{self.controller.outpost.type.lower()}",
                 "app.kubernetes.io/instance": slugify(self.controller.outpost.name),
-                "app.kubernetes.io/version": __version__,
                 "app.kubernetes.io/managed-by": "goauthentik.io",
-                "goauthentik.io/outpost-uuid": self.controller.outpost.uuid.hex,
-                "goauthentik.io/outpost-type": str(self.controller.outpost.type),
+                "app.kubernetes.io/name": f"authentik-{self.controller.outpost.type.lower()}",
+                "app.kubernetes.io/version": get_version(),
                 "goauthentik.io/outpost-name": slugify(self.controller.outpost.name),
+                "goauthentik.io/outpost-type": str(self.controller.outpost.type),
+                "goauthentik.io/outpost-uuid": self.controller.outpost.uuid.hex,
             },
             **kwargs,
         )
diff --git a/tests/integration/test_outpost_kubernetes.py b/tests/integration/test_outpost_kubernetes.py
index b6d7795f2..37e386849 100644
--- a/tests/integration/test_outpost_kubernetes.py
+++ b/tests/integration/test_outpost_kubernetes.py
@@ -1,5 +1,9 @@
 """outpost tests"""
+from unittest.mock import MagicMock, patch
+
 from django.test import TestCase
+from kubernetes.client import AppsV1Api
+from kubernetes.client.exceptions import OpenApiException
 
 from authentik.core.tests.utils import create_test_flow
 from authentik.lib.config import CONFIG
@@ -58,3 +62,29 @@ class OutpostKubernetesTests(TestCase):
                 )
 
         deployment_reconciler.delete(deployment_reconciler.get_reference_object())
+
+    def test_controller_rename(self):
+        """test that objects get deleted and re-created with new names"""
+        controller = ProxyKubernetesController(self.outpost, self.service_connection)
+
+        self.assertIsNone(controller.up())
+        self.outpost.name = "foo"
+        self.assertIsNone(controller.up())
+        apps = AppsV1Api(controller.client)
+        with self.assertRaises(OpenApiException):
+            apps.read_namespaced_deployment("test", self.outpost.config.kubernetes_namespace)
+        controller.down()
+
+    def test_controller_full_update(self):
+        """Test an update that triggers all objects"""
+        controller = ProxyKubernetesController(self.outpost, self.service_connection)
+
+        self.assertIsNone(controller.up())
+        with patch(
+            "authentik.outposts.controllers.k8s.base.get_version", MagicMock(return_value="1234")
+        ):
+            self.assertIsNone(controller.up())
+        deployment_reconciler = DeploymentReconciler(controller)
+        deployment = deployment_reconciler.retrieve()
+        self.assertEqual(deployment.metadata.labels["app.kubernetes.io/version"], "1234")
+        controller.down()
diff --git a/tests/setup.sh b/tests/setup.sh
deleted file mode 100755
index a16edc519..000000000
--- a/tests/setup.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash -x
-# Setup docker & compose
-curl -fsSL https://get.docker.com | bash
-sudo usermod -a -G docker ubuntu
-sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-sudo chmod +x /usr/local/bin/docker-compose
-# Setup nodejs
-curl -sL https://deb.nodesource.com/setup_15.x | sudo -E bash -
-sudo apt-get install -y nodejs
-# Setup python
-sudo apt install -y python3.9 python3.9-dev python3-pip libxmlsec1-dev pkg-config
-# Setup docker
-sudo pip3 install pipenv
-
-cd tests/e2e
-sudo docker-compose up -d
-cd ../..
-pipenv sync --dev