internal: add internal healthchecking to prevent websocket errors

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

cmd/server/main.go

@@ -54,7 +54,7 @@ func main() {
 	u, _ := url.Parse("http://localhost:8000")
 
 	g := gounicorn.NewGoUnicorn()
-	ws := web.NewWebServer()
+	ws := web.NewWebServer(g)
 	defer g.Kill()
 	defer ws.Shutdown()
 	go web.RunMetricsServer()

internal/gounicorn/gounicorn.go

@@ -1,10 +1,13 @@
 package gounicorn
 
 import (
+	"net/http"
 	"os"
 	"os/exec"
+	"time"
 
 	log "github.com/sirupsen/logrus"
+	"goauthentik.io/internal/outpost/ak"
 )
 
 type GoUnicorn struct {
@@ -12,6 +15,7 @@ type GoUnicorn struct {
 	p       *exec.Cmd
 	started bool
 	killed  bool
+	alive   bool
 }
 
 func NewGoUnicorn() *GoUnicorn {
@@ -20,6 +24,7 @@ func NewGoUnicorn() *GoUnicorn {
 		log:     logger,
 		started: false,
 		killed:  false,
+		alive:   false,
 	}
 	g.initCmd()
 	return g
@@ -35,6 +40,10 @@ func (g *GoUnicorn) initCmd() {
 	g.p.Stderr = os.Stderr
 }
 
+func (g *GoUnicorn) IsRunning() bool {
+	return g.alive
+}
+
 func (g *GoUnicorn) Start() error {
 	if g.killed {
 		g.log.Debug("Not restarting gunicorn since we're killed")
@@ -44,9 +53,38 @@ func (g *GoUnicorn) Start() error {
 		g.initCmd()
 	}
 	g.started = true
+	go g.healthcheck()
 	return g.p.Run()
 }
 
+func (g *GoUnicorn) healthcheck() {
+	g.log.Debug("starting healthcheck")
+	h := &http.Client{
+		Transport: ak.NewUserAgentTransport("goauthentik.io go proxy healthcheck", http.DefaultTransport),
+	}
+	check := func() bool {
+		res, err := h.Get("http://localhost:8000/-/health/live/")
+		if err == nil && res.StatusCode == 204 {
+			g.alive = true
+			return true
+		}
+		return false
+	}
+
+	// Default healthcheck is every 1 second on startup
+	// once we've been healthy once, increase to 30 seconds
+	for range time.Tick(time.Second) {
+		if check() {
+			g.log.Info("backend is alive, backing off with healthchecks")
+			break
+		}
+		g.log.Debug("backend not alive yet")
+	}
+	for range time.Tick(30 * time.Second) {
+		check()
+	}
+}
+
 func (g *GoUnicorn) Kill() {
 	g.killed = true
 	err := g.p.Process.Kill()

internal/web/proxy.go

@@ -40,6 +40,10 @@ func (ws *WebServer) configureProxy() {
 		ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running"))
 	})
 	ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		if !ws.p.IsRunning() {
+			ws.proxyErrorHandler(rw, r, fmt.Errorf("authentik core not running yet"))
+			return
+		}
 		host := web.GetHost(r)
 		before := time.Now()
 		if ws.ProxyServer != nil {
@@ -59,8 +63,12 @@ func (ws *WebServer) configureProxy() {
 }
 
 func (ws *WebServer) proxyErrorHandler(rw http.ResponseWriter, req *http.Request, err error) {
-	ws.log.WithError(err).Warning("proxy error")
+	ws.log.Warning(err.Error())
 	rw.WriteHeader(http.StatusBadGateway)
+	_, err = rw.Write([]byte("authentik starting..."))
+	if err != nil {
+		ws.log.WithError(err).Warning("failed to write error message")
+	}
 }
 
 func (ws *WebServer) proxyModifyResponse(r *http.Response) error {

internal/web/web.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pires/go-proxyproto"
 	log "github.com/sirupsen/logrus"
 	"goauthentik.io/internal/config"
+	"goauthentik.io/internal/gounicorn"
 	"goauthentik.io/internal/outpost/proxyv2"
 )
 
@@ -27,9 +28,10 @@ type WebServer struct {
 	m   *mux.Router
 	lh  *mux.Router
 	log *log.Entry
+	p   *gounicorn.GoUnicorn
 }
 
-func NewWebServer() *WebServer {
+func NewWebServer(g *gounicorn.GoUnicorn) *WebServer {
 	l := log.WithField("logger", "authentik.g.web")
 	mainHandler := mux.NewRouter()
 	if config.G.ErrorReporting.Enabled {
@@ -46,6 +48,7 @@ func NewWebServer() *WebServer {
 		m:   mainHandler,
 		lh:  logginRouter,
 		log: l,
+		p:   g,
 	}
 	ws.configureStatic()
 	ws.configureProxy()

lifecycle/ak

@@ -49,7 +49,7 @@ elif [[ "$1" == "test" ]]; then
 elif [[ "$1" == "healthcheck" ]]; then
     mode=$(cat $MODE_FILE)
     if [[ $mode == "server" ]]; then
-        curl --user-agent "authentik Healthcheck" -I http://localhost:9000/-/health/ready/
+        curl --user-agent "goauthentik.io lifecycle Healthcheck" -I http://localhost:9000/-/health/ready/
     elif [[ $mode == "worker" ]]; then
         celery -A authentik.root.celery inspect ping -d celery@$HOSTNAME
     fi