core: metrics v2 (#1370)

* outposts: add ldap metrics, move ping to 9100

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

* outpost: add flow_executor metrics

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

* use port 9300 for metrics, add core metrics port

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

* outposts/controllers/k8s: add service monitor creation support

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

authentik/lib/default.yml

@@ -9,6 +9,7 @@ postgresql:
 web:
   listen: 0.0.0.0:9000
   listen_tls: 0.0.0.0:9443
+  listen_metrics: 0.0.0.0:9100
   load_local_files: false
   outpost_port_offset: 0
 

authentik/outposts/controllers/k8s/service_monitor.py

@@ -0,0 +1,150 @@
+"""Kubernetes Prometheus ServiceMonitor Reconciler"""
+from dataclasses import asdict, dataclass, field
+from typing import TYPE_CHECKING
+
+from dacite import from_dict
+from kubernetes.client import ApiextensionsV1Api, CustomObjectsApi
+
+from authentik.outposts.controllers.base import FIELD_MANAGER
+from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler
+
+if TYPE_CHECKING:
+    from authentik.outposts.controllers.kubernetes import KubernetesController
+
+
+@dataclass
+class PrometheusServiceMonitorSpecEndpoint:
+    """Prometheus ServiceMonitor endpoint spec"""
+
+    port: str
+    path: str = field(default="/metrics")
+
+
+@dataclass
+class PrometheusServiceMonitorSpecSelector:
+    """Prometheus ServiceMonitor selector spec"""
+
+    # pylint: disable=invalid-name
+    matchLabels: dict
+
+
+@dataclass
+class PrometheusServiceMonitorSpec:
+    """Prometheus ServiceMonitor spec"""
+
+    endpoints: list[PrometheusServiceMonitorSpecEndpoint]
+    # pylint: disable=invalid-name
+    selector: PrometheusServiceMonitorSpecSelector
+
+
+@dataclass
+class PrometheusServiceMonitorMetadata:
+    """Prometheus ServiceMonitor metadata"""
+
+    name: str
+    namespace: str
+    labels: dict = field(default_factory=dict)
+
+
+@dataclass
+class PrometheusServiceMonitor:
+    """Prometheus ServiceMonitor"""
+
+    # pylint: disable=invalid-name
+    apiVersion: str
+    kind: str
+    metadata: PrometheusServiceMonitorMetadata
+    spec: PrometheusServiceMonitorSpec
+
+
+CRD_NAME = "servicemonitors.monitoring.coreos.com"
+CRD_GROUP = "monitoring.coreos.com"
+CRD_VERSION = "v1"
+CRD_PLURAL = "servicemonitors"
+
+
+class PrometheusServiceMonitorReconciler(KubernetesObjectReconciler[PrometheusServiceMonitor]):
+    """Kubernetes Prometheus ServiceMonitor Reconciler"""
+
+    def __init__(self, controller: "KubernetesController") -> None:
+        super().__init__(controller)
+        self.api_ex = ApiextensionsV1Api(controller.client)
+        self.api = CustomObjectsApi(controller.client)
+
+    @property
+    def noop(self) -> bool:
+        return not self._crd_exists()
+
+    def _crd_exists(self) -> bool:
+        """Check if the Prometheus ServiceMonitor exists"""
+        return bool(
+            len(
+                self.api_ex.list_custom_resource_definition(
+                    field_selector=f"metadata.name={CRD_NAME}"
+                ).items
+            )
+        )
+
+    def get_reference_object(self) -> PrometheusServiceMonitor:
+        """Get service monitor object for outpost"""
+        return PrometheusServiceMonitor(
+            apiVersion=f"{CRD_GROUP}/{CRD_VERSION}",
+            kind="ServiceMonitor",
+            metadata=PrometheusServiceMonitorMetadata(
+                name=self.name,
+                namespace=self.namespace,
+                labels=self.get_object_meta().labels,
+            ),
+            spec=PrometheusServiceMonitorSpec(
+                endpoints=[
+                    PrometheusServiceMonitorSpecEndpoint(
+                        port="http-metrics",
+                    )
+                ],
+                selector=PrometheusServiceMonitorSpecSelector(
+                    matchLabels=self.get_object_meta(name=self.name).labels,
+                ),
+            ),
+        )
+
+    def create(self, reference: PrometheusServiceMonitor):
+        return self.api.create_namespaced_custom_object(
+            group=CRD_GROUP,
+            version=CRD_VERSION,
+            plural=CRD_PLURAL,
+            namespace=self.namespace,
+            body=asdict(reference),
+            field_manager=FIELD_MANAGER,
+        )
+
+    def delete(self, reference: PrometheusServiceMonitor):
+        return self.api.delete_namespaced_custom_object(
+            group=CRD_GROUP,
+            version=CRD_VERSION,
+            namespace=self.namespace,
+            plural=CRD_PLURAL,
+            name=self.name,
+        )
+
+    def retrieve(self) -> PrometheusServiceMonitor:
+        return from_dict(
+            PrometheusServiceMonitor,
+            self.api.get_namespaced_custom_object(
+                group=CRD_GROUP,
+                version=CRD_VERSION,
+                namespace=self.namespace,
+                plural=CRD_PLURAL,
+                name=self.name,
+            ),
+        )
+
+    def update(self, current: PrometheusServiceMonitor, reference: PrometheusServiceMonitor):
+        return self.api.patch_namespaced_custom_object(
+            group=CRD_GROUP,
+            version=CRD_VERSION,
+            namespace=self.namespace,
+            plural=CRD_PLURAL,
+            name=self.name,
+            body=asdict(reference),
+            field_manager=FIELD_MANAGER,
+        )

authentik/outposts/controllers/kubernetes.py

@@ -13,6 +13,7 @@ from authentik.outposts.controllers.k8s.base import KubernetesObjectReconciler
 from authentik.outposts.controllers.k8s.deployment import DeploymentReconciler
 from authentik.outposts.controllers.k8s.secret import SecretReconciler
 from authentik.outposts.controllers.k8s.service import ServiceReconciler
+from authentik.outposts.controllers.k8s.service_monitor import PrometheusServiceMonitorReconciler
 from authentik.outposts.models import KubernetesServiceConnection, Outpost, ServiceConnectionInvalid
 
 
@@ -32,8 +33,9 @@ class KubernetesController(BaseController):
             "secret": SecretReconciler,
             "deployment": DeploymentReconciler,
             "service": ServiceReconciler,
+            "prometheus servicemonitor": PrometheusServiceMonitorReconciler,
         }
-        self.reconcile_order = ["secret", "deployment", "service"]
+        self.reconcile_order = ["secret", "deployment", "service", "prometheus servicemonitor"]
 
     def up(self):
         try:

authentik/providers/ldap/controllers/docker.py

@@ -12,4 +12,5 @@ class LDAPDockerController(DockerController):
         self.deployment_ports = [
             DeploymentPort(389, "ldap", "tcp", 3389),
             DeploymentPort(636, "ldaps", "tcp", 6636),
+            DeploymentPort(9300, "http-metrics", "tcp", 9300),
         ]

authentik/providers/ldap/controllers/kubernetes.py

@@ -12,4 +12,5 @@ class LDAPKubernetesController(KubernetesController):
         self.deployment_ports = [
             DeploymentPort(389, "ldap", "tcp", 3389),
             DeploymentPort(636, "ldaps", "tcp", 6636),
+            DeploymentPort(9300, "http-metrics", "tcp", 9300),
         ]

authentik/providers/proxy/controllers/docker.py

@@ -14,7 +14,7 @@ class ProxyDockerController(DockerController):
         super().__init__(outpost, connection)
         self.deployment_ports = [
             DeploymentPort(9000, "http", "tcp"),
-            DeploymentPort(9100, "http-metrics", "tcp"),
+            DeploymentPort(9300, "http-metrics", "tcp"),
             DeploymentPort(9443, "https", "tcp"),
         ]
 

authentik/providers/proxy/controllers/kubernetes.py

@@ -13,7 +13,7 @@ class ProxyKubernetesController(KubernetesController):
         super().__init__(outpost, connection)
         self.deployment_ports = [
             DeploymentPort(9000, "http", "tcp"),
-            DeploymentPort(9100, "http-metrics", "tcp"),
+            DeploymentPort(9300, "http-metrics", "tcp"),
             DeploymentPort(9443, "https", "tcp"),
         ]
         self.reconcilers["ingress"] = IngressReconciler

cmd/server/main.go

@@ -57,6 +57,7 @@ func main() {
 	ws := web.NewWebServer()
 	defer g.Kill()
 	defer ws.Shutdown()
+	go web.RunMetricsServer()
 	for {
 		go attemptStartBackend(g)
 		ws.Start()

internal/config/struct.go

@@ -29,6 +29,7 @@ type RedisConfig struct {
 type WebConfig struct {
 	Listen                 string `yaml:"listen"`
 	ListenTLS              string `yaml:"listen_tls"`
+	ListenMetrics          string `yaml:"listen_metrics"`
 	LoadLocalFiles         bool   `yaml:"load_local_files" env:"AUTHENTIK_WEB_LOAD_LOCAL_FILES"`
 	DisableEmbeddedOutpost bool   `yaml:"disable_embedded_outpost" env:"AUTHENTIK_WEB__DISABLE_EMBEDDED_OUTPOST"`
 }

internal/outpost/flow.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
 	"goauthentik.io/api"
 	"goauthentik.io/internal/constants"
@@ -21,6 +23,17 @@ import (
 
 type StageComponent string
 
+var (
+	FlowTimingGet = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "authentik_outpost_flow_timing_get",
+		Help: "Duration it took to get a challenge",
+	}, []string{"stage", "flow", "client", "user"})
+	FlowTimingPost = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "authentik_outpost_flow_timing_post",
+		Help: "Duration it took to send a challenge",
+	}, []string{"stage", "flow", "client", "user"})
+)
+
 const (
 	StageIdentification        = StageComponent("ak-stage-identification")
 	StagePassword              = StageComponent("ak-stage-password")
@@ -38,6 +51,7 @@ type FlowExecutor struct {
 	Answers map[StageComponent]string
 	Context context.Context
 
+	cip      string
 	api      *api.APIClient
 	flowSlug string
 	log      *log.Entry
@@ -75,6 +89,7 @@ func NewFlowExecutor(ctx context.Context, flowSlug string, refConfig *api.Config
 		log:      l,
 		token:    token,
 		sp:       rsp,
+		cip:      "",
 	}
 }
 
@@ -89,7 +104,8 @@ type ChallengeInt interface {
 }
 
 func (fe *FlowExecutor) DelegateClientIP(a net.Addr) {
-	fe.api.GetConfig().AddDefaultHeader(HeaderAuthentikRemoteIP, utils.GetIP(a))
+	fe.cip = utils.GetIP(a)
+	fe.api.GetConfig().AddDefaultHeader(HeaderAuthentikRemoteIP, fe.cip)
 }
 
 func (fe *FlowExecutor) CheckApplicationAccess(appSlug string) (bool, error) {
@@ -142,6 +158,12 @@ func (fe *FlowExecutor) solveFlowChallenge(depth int) (bool, error) {
 	gcsp.SetTag("ak_challenge", string(ch.GetType()))
 	gcsp.SetTag("ak_component", ch.GetComponent())
 	gcsp.Finish()
+	FlowTimingGet.With(prometheus.Labels{
+		"stage":  ch.GetComponent(),
+		"flow":   fe.flowSlug,
+		"client": fe.cip,
+		"user":   fe.Answers[StageIdentification],
+	}).Observe(float64(gcsp.EndTime.Sub(gcsp.StartTime)))
 
 	// Resole challenge
 	scsp := sentry.StartSpan(fe.Context, "authentik.outposts.flow_executor.solve_challenge")
@@ -202,6 +224,13 @@ func (fe *FlowExecutor) solveFlowChallenge(depth int) (bool, error) {
 			}
 		}
 	}
+	FlowTimingPost.With(prometheus.Labels{
+		"stage":  ch.GetComponent(),
+		"flow":   fe.flowSlug,
+		"client": fe.cip,
+		"user":   fe.Answers[StageIdentification],
+	}).Observe(float64(scsp.EndTime.Sub(scsp.StartTime)))
+
 	if depth >= 10 {
 		return false, errors.New("exceeded stage recursion depth")
 	}

internal/outpost/ldap/api.go

@@ -6,13 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"net/http"
 	"strings"
 	"sync"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/pires/go-proxyproto"
 	log "github.com/sirupsen/logrus"
+	"goauthentik.io/internal/outpost/ldap/metrics"
 )
 
 const (
@@ -65,16 +65,6 @@ func (ls *LDAPServer) Refresh() error {
 	return nil
 }
 
-func (ls *LDAPServer) StartHTTPServer() error {
-	listen := "0.0.0.0:9000" // same port as proxy
-	m := http.NewServeMux()
-	m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) {
-		rw.WriteHeader(204)
-	})
-	ls.log.WithField("listen", listen).Info("Starting http server")
-	return http.ListenAndServe(listen, m)
-}
-
 func (ls *LDAPServer) StartLDAPServer() error {
 	listen := "0.0.0.0:3389"
 
@@ -126,10 +116,7 @@ func (ls *LDAPServer) Start() error {
 	wg.Add(3)
 	go func() {
 		defer wg.Done()
-		err := ls.StartHTTPServer()
+		metrics.RunServer()
-		if err != nil {
-			panic(err)
-		}
 	}()
 	go func() {
 		defer wg.Done()

internal/outpost/ldap/bind.go

@@ -8,7 +8,9 @@ import (
 	"github.com/getsentry/sentry-go"
 	"github.com/google/uuid"
 	"github.com/nmcclain/ldap"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+	"goauthentik.io/internal/outpost/ldap/metrics"
 	"goauthentik.io/internal/utils"
 )
 
@@ -27,7 +29,6 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD
 	rid := uuid.New().String()
 	span.SetTag("request_uid", rid)
 	span.SetTag("user.username", bindDN)
-	defer span.Finish()
 
 	bindDN = strings.ToLower(bindDN)
 	req := BindRequest{
@@ -38,7 +39,16 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD
 		id:     rid,
 		ctx:    span.Context(),
 	}
-	req.log.Info("Bind request")
+	defer func() {
+		span.Finish()
+		metrics.Requests.With(prometheus.Labels{
+			"type":   "bind",
+			"filter": "",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Observe(float64(span.EndTime.Sub(span.StartTime)))
+		req.log.WithField("took-ms", span.EndTime.Sub(span.StartTime).Milliseconds()).Info("Bind request")
+	}()
 	for _, instance := range ls.providers {
 		username, err := instance.getUsername(bindDN)
 		if err == nil {
@@ -48,6 +58,12 @@ func (ls *LDAPServer) Bind(bindDN string, bindPW string, conn net.Conn) (ldap.LD
 		}
 	}
 	req.log.WithField("request", "bind").Warning("No provider found for request")
+	metrics.RequestsRejected.With(prometheus.Labels{
+		"type":   "bind",
+		"reason": "no_provider",
+		"dn":     bindDN,
+		"client": utils.GetIP(conn.RemoteAddr()),
+	}).Inc()
 	return ldap.LDAPResultOperationsError, nil
 }
 

internal/outpost/ldap/instance_bind.go

@@ -8,9 +8,11 @@ import (
 	"github.com/getsentry/sentry-go"
 	goldap "github.com/go-ldap/ldap/v3"
 	"github.com/nmcclain/ldap"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"goauthentik.io/api"
 	"goauthentik.io/internal/outpost"
+	"goauthentik.io/internal/outpost/ldap/metrics"
 	"goauthentik.io/internal/utils"
 )
 
@@ -48,9 +50,21 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes
 
 	passed, err := fe.Execute()
 	if !passed {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "bind",
+			"reason": "invalid_credentials",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.LDAPResultInvalidCredentials, nil
 	}
 	if err != nil {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "bind",
+			"reason": "flow_error",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		req.log.WithError(err).Warning("failed to execute flow")
 		return ldap.LDAPResultOperationsError, nil
 	}
@@ -58,9 +72,21 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes
 	access, err := fe.CheckApplicationAccess(pi.appSlug)
 	if !access {
 		req.log.Info("Access denied for user")
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "bind",
+			"reason": "access_denied",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.LDAPResultInsufficientAccessRights, nil
 	}
 	if err != nil {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "bind",
+			"reason": "access_check_fail",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		req.log.WithError(err).Warning("failed to check access")
 		return ldap.LDAPResultOperationsError, nil
 	}
@@ -69,6 +95,12 @@ func (pi *ProviderInstance) Bind(username string, req BindRequest) (ldap.LDAPRes
 	// Get user info to store in context
 	userInfo, _, err := fe.ApiClient().CoreApi.CoreUsersMeRetrieve(context.Background()).Execute()
 	if err != nil {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "bind",
+			"reason": "user_info_fail",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		req.log.WithError(err).Warning("failed to get user info")
 		return ldap.LDAPResultOperationsError, nil
 	}

internal/outpost/ldap/instance_search.go

@@ -8,7 +8,10 @@ import (
 
 	"github.com/getsentry/sentry-go"
 	"github.com/nmcclain/ldap"
+	"github.com/prometheus/client_golang/prometheus"
 	"goauthentik.io/api"
+	"goauthentik.io/internal/outpost/ldap/metrics"
+	"goauthentik.io/internal/utils"
 )
 
 func (pi *ProviderInstance) SearchMe(req SearchRequest, f UserFlags) (ldap.ServerSearchResult, error) {
@@ -32,12 +35,30 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult,
 	entries := []*ldap.Entry{}
 	filterEntity, err := ldap.GetFilterObjectClass(req.Filter)
 	if err != nil {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "filter_parse_fail",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: error parsing filter: %s", req.Filter)
 	}
 	if len(req.BindDN) < 1 {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "empty_bind_dn",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, fmt.Errorf("Search Error: Anonymous BindDN not allowed %s", req.BindDN)
 	}
 	if !strings.HasSuffix(req.BindDN, baseDN) {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "invalid_bind_dn",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, fmt.Errorf("Search Error: BindDN %s not in our BaseDN %s", req.BindDN, pi.BaseDN)
 	}
 
@@ -46,6 +67,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult,
 	pi.boundUsersMutex.RUnlock()
 	if !ok {
 		pi.log.Debug("User info not cached")
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "user_info_not_cached",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultInsufficientAccessRights}, errors.New("access denied")
 	}
 	if !flags.CanSearch {
@@ -56,6 +83,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult,
 
 	parsedFilter, err := ldap.CompileFilter(req.Filter)
 	if err != nil {
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "filter_parse_fail",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: error parsing filter: %s", req.Filter)
 	}
 
@@ -65,6 +98,12 @@ func (pi *ProviderInstance) Search(req SearchRequest) (ldap.ServerSearchResult,
 
 	switch filterEntity {
 	default:
+		metrics.RequestsRejected.With(prometheus.Labels{
+			"type":   "search",
+			"reason": "unhandled_filter_type",
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Inc()
 		return ldap.ServerSearchResult{ResultCode: ldap.LDAPResultOperationsError}, fmt.Errorf("Search Error: unhandled filter type: %s [%s]", filterEntity, req.Filter)
 	case GroupObjectClass:
 		wg := sync.WaitGroup{}

internal/outpost/ldap/metrics/metrics.go

@@ -0,0 +1,33 @@
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var (
+	Requests = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "authentik_outpost_ldap_requests",
+		Help: "The total number of configured providers",
+	}, []string{"type", "dn", "filter", "client"})
+	RequestsRejected = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "authentik_outpost_ldap_requests_rejected",
+		Help: "Total number of rejected requests",
+	}, []string{"type", "reason", "dn", "client"})
+)
+
+func RunServer() {
+	m := mux.NewRouter()
+	m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) {
+		rw.WriteHeader(204)
+	})
+	m.Path("/metrics").Handler(promhttp.Handler())
+	err := http.ListenAndServe("0.0.0.0:9300", m)
+	if err != nil {
+		panic(err)
+	}
+}

internal/outpost/ldap/search.go

@@ -10,7 +10,9 @@ import (
 	goldap "github.com/go-ldap/ldap/v3"
 	"github.com/google/uuid"
 	"github.com/nmcclain/ldap"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+	"goauthentik.io/internal/outpost/ldap/metrics"
 	"goauthentik.io/internal/utils"
 )
 
@@ -43,6 +45,12 @@ func (ls *LDAPServer) Search(bindDN string, searchReq ldap.SearchRequest, conn n
 
 	defer func() {
 		span.Finish()
+		metrics.Requests.With(prometheus.Labels{
+			"type":   "search",
+			"filter": req.Filter,
+			"dn":     req.BindDN,
+			"client": utils.GetIP(req.conn.RemoteAddr()),
+		}).Observe(float64(span.EndTime.Sub(span.StartTime)))
 		req.log.WithField("took-ms", span.EndTime.Sub(span.StartTime).Milliseconds()).Info("Search request")
 	}()
 

internal/outpost/proxyv2/metrics/metrics.go

@@ -22,8 +22,11 @@ var (
 
 func RunServer() {
 	m := mux.NewRouter()
+	m.HandleFunc("/akprox/ping", func(rw http.ResponseWriter, r *http.Request) {
+		rw.WriteHeader(204)
+	})
 	m.Path("/metrics").Handler(promhttp.Handler())
-	err := http.ListenAndServe("localhost:9300", m)
+	err := http.ListenAndServe("0.0.0.0:9300", m)
 	if err != nil {
 		panic(err)
 	}

internal/outpost/proxyv2/proxyv2.go

@@ -63,7 +63,6 @@ func NewProxyServer(ac *ak.APIController, portOffset int) *ProxyServer {
 		akAPI:       ac,
 		defaultCert: defaultCert,
 	}
-	globalMux.Path("/akprox/ping").HandlerFunc(s.HandlePing)
 	globalMux.PathPrefix("/akprox/static").HandlerFunc(s.HandleStatic)
 	rootMux.PathPrefix("/").HandlerFunc(s.Handle)
 	return s

internal/web/metrics.go

@@ -0,0 +1,58 @@
+package web
+
+import (
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"goauthentik.io/internal/config"
+)
+
+var (
+	Requests = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "authentik_main_requests",
+		Help: "The total number of configured providers",
+	}, []string{"dest"})
+)
+
+func RunMetricsServer() {
+	m := mux.NewRouter()
+	m.Path("/metrics").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		defer promhttp.InstrumentMetricHandler(
+			prometheus.DefaultRegisterer, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{
+				DisableCompression: true,
+			}),
+		).ServeHTTP(rw, r)
+
+		// Get upstream metrics
+		re, err := http.NewRequest("GET", "http://localhost:8000/metrics/", nil)
+		if err != nil {
+			log.WithError(err).Warning("failed to get upstream metrics")
+			return
+		}
+		re.SetBasicAuth("monitor", config.G.SecretKey)
+		res, err := http.DefaultClient.Do(re)
+		if err != nil {
+			log.WithError(err).Warning("failed to get upstream metrics")
+			return
+		}
+		bm, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			log.WithError(err).Warning("failed to get upstream metrics")
+			return
+		}
+		_, err = rw.Write(bm)
+		if err != nil {
+			log.WithError(err).Warning("failed to get upstream metrics")
+			return
+		}
+	})
+	err := http.ListenAndServe(config.G.Web.ListenMetrics, m)
+	if err != nil {
+		panic(err)
+	}
+}

internal/web/proxy.go

@@ -5,7 +5,9 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"goauthentik.io/internal/utils/web"
 )
 
@@ -28,18 +30,29 @@ func (ws *WebServer) configureProxy() {
 	rp.ModifyResponse = ws.proxyModifyResponse
 	ws.m.PathPrefix("/akprox").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		if ws.ProxyServer != nil {
+			before := time.Now()
 			ws.ProxyServer.Handle(rw, r)
+			Requests.With(prometheus.Labels{
+				"dest": "embedded_outpost",
+			}).Observe(float64(time.Since(before)))
 			return
 		}
 		ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running"))
 	})
 	ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 		host := web.GetHost(r)
+		before := time.Now()
 		if ws.ProxyServer != nil {
 			if ws.ProxyServer.HandleHost(host, rw, r) {
+				Requests.With(prometheus.Labels{
+					"dest": "embedded_outpost",
+				}).Observe(float64(time.Since(before)))
 				return
 			}
 		}
+		Requests.With(prometheus.Labels{
+			"dest": "py",
+		}).Observe(float64(time.Since(before)))
 		ws.log.WithField("host", host).Trace("routing to application server")
 		rp.ServeHTTP(rw, r)
 	})

ldap.Dockerfile

@@ -31,6 +31,6 @@ ENV GIT_BUILD_HASH=$GIT_BUILD_HASH
 
 COPY --from=builder /go/ldap /
 
-HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9000/akprox/ping" ]
+HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9300/akprox/ping" ]
 
 ENTRYPOINT ["/ldap"]

proxy.Dockerfile

@@ -43,6 +43,6 @@ ENV GIT_BUILD_HASH=$GIT_BUILD_HASH
 
 COPY --from=builder /go/proxy /
 
-HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9000/akprox/ping" ]
+HEALTHCHECK CMD [ "wget", "--spider", "http://localhost:9300/akprox/ping" ]
 
 ENTRYPOINT ["/proxy"]

website/docs/outposts/outposts.md

@@ -61,6 +61,7 @@ kubernetes_service_type: ClusterIP
 # - 'secret'
 # - 'deployment'
 # - 'service'
+# - 'prometheus servicemonitor'
 # - 'ingress'
 # - 'traefik middleware'
 kubernetes_disabled_components: []