internal: add internal healthchecking to prevent websocket errors

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>
This commit is contained in:
Jens Langhammer 2021-10-05 22:19:33 +02:00
parent 40055ef01b
commit 2aacb311bc
5 changed files with 53 additions and 4 deletions

View File

@ -54,7 +54,7 @@ func main() {
u, _ := url.Parse("http://localhost:8000") u, _ := url.Parse("http://localhost:8000")
g := gounicorn.NewGoUnicorn() g := gounicorn.NewGoUnicorn()
ws := web.NewWebServer() ws := web.NewWebServer(g)
defer g.Kill() defer g.Kill()
defer ws.Shutdown() defer ws.Shutdown()
go web.RunMetricsServer() go web.RunMetricsServer()

View File

@ -1,10 +1,13 @@
package gounicorn package gounicorn
import ( import (
"net/http"
"os" "os"
"os/exec" "os/exec"
"time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"goauthentik.io/internal/outpost/ak"
) )
type GoUnicorn struct { type GoUnicorn struct {
@ -12,6 +15,7 @@ type GoUnicorn struct {
p *exec.Cmd p *exec.Cmd
started bool started bool
killed bool killed bool
alive bool
} }
func NewGoUnicorn() *GoUnicorn { func NewGoUnicorn() *GoUnicorn {
@ -20,6 +24,7 @@ func NewGoUnicorn() *GoUnicorn {
log: logger, log: logger,
started: false, started: false,
killed: false, killed: false,
alive: false,
} }
g.initCmd() g.initCmd()
return g return g
@ -35,6 +40,10 @@ func (g *GoUnicorn) initCmd() {
g.p.Stderr = os.Stderr g.p.Stderr = os.Stderr
} }
func (g *GoUnicorn) IsRunning() bool {
return g.alive
}
func (g *GoUnicorn) Start() error { func (g *GoUnicorn) Start() error {
if g.killed { if g.killed {
g.log.Debug("Not restarting gunicorn since we're killed") g.log.Debug("Not restarting gunicorn since we're killed")
@ -44,9 +53,38 @@ func (g *GoUnicorn) Start() error {
g.initCmd() g.initCmd()
} }
g.started = true g.started = true
go g.healthcheck()
return g.p.Run() return g.p.Run()
} }
func (g *GoUnicorn) healthcheck() {
g.log.Debug("starting healthcheck")
h := &http.Client{
Transport: ak.NewUserAgentTransport("goauthentik.io go proxy healthcheck", http.DefaultTransport),
}
check := func() bool {
res, err := h.Get("http://localhost:8000/-/health/live/")
if err == nil && res.StatusCode == 204 {
g.alive = true
return true
}
return false
}
// Default healthcheck is every 1 second on startup
// once we've been healthy once, increase to 30 seconds
for range time.Tick(time.Second) {
if check() {
g.log.Info("backend is alive, backing off with healthchecks")
break
}
g.log.Debug("backend not alive yet")
}
for range time.Tick(30 * time.Second) {
check()
}
}
func (g *GoUnicorn) Kill() { func (g *GoUnicorn) Kill() {
g.killed = true g.killed = true
err := g.p.Process.Kill() err := g.p.Process.Kill()

View File

@ -40,6 +40,10 @@ func (ws *WebServer) configureProxy() {
ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running")) ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running"))
}) })
ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
if !ws.p.IsRunning() {
ws.proxyErrorHandler(rw, r, fmt.Errorf("authentik core not running yet"))
return
}
host := web.GetHost(r) host := web.GetHost(r)
before := time.Now() before := time.Now()
if ws.ProxyServer != nil { if ws.ProxyServer != nil {
@ -59,8 +63,12 @@ func (ws *WebServer) configureProxy() {
} }
func (ws *WebServer) proxyErrorHandler(rw http.ResponseWriter, req *http.Request, err error) { func (ws *WebServer) proxyErrorHandler(rw http.ResponseWriter, req *http.Request, err error) {
ws.log.WithError(err).Warning("proxy error") ws.log.Warning(err.Error())
rw.WriteHeader(http.StatusBadGateway) rw.WriteHeader(http.StatusBadGateway)
_, err = rw.Write([]byte("authentik starting..."))
if err != nil {
ws.log.WithError(err).Warning("failed to write error message")
}
} }
func (ws *WebServer) proxyModifyResponse(r *http.Response) error { func (ws *WebServer) proxyModifyResponse(r *http.Response) error {

View File

@ -11,6 +11,7 @@ import (
"github.com/pires/go-proxyproto" "github.com/pires/go-proxyproto"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"goauthentik.io/internal/config" "goauthentik.io/internal/config"
"goauthentik.io/internal/gounicorn"
"goauthentik.io/internal/outpost/proxyv2" "goauthentik.io/internal/outpost/proxyv2"
) )
@ -27,9 +28,10 @@ type WebServer struct {
m *mux.Router m *mux.Router
lh *mux.Router lh *mux.Router
log *log.Entry log *log.Entry
p *gounicorn.GoUnicorn
} }
func NewWebServer() *WebServer { func NewWebServer(g *gounicorn.GoUnicorn) *WebServer {
l := log.WithField("logger", "authentik.g.web") l := log.WithField("logger", "authentik.g.web")
mainHandler := mux.NewRouter() mainHandler := mux.NewRouter()
if config.G.ErrorReporting.Enabled { if config.G.ErrorReporting.Enabled {
@ -46,6 +48,7 @@ func NewWebServer() *WebServer {
m: mainHandler, m: mainHandler,
lh: logginRouter, lh: logginRouter,
log: l, log: l,
p: g,
} }
ws.configureStatic() ws.configureStatic()
ws.configureProxy() ws.configureProxy()

View File

@ -49,7 +49,7 @@ elif [[ "$1" == "test" ]]; then
elif [[ "$1" == "healthcheck" ]]; then elif [[ "$1" == "healthcheck" ]]; then
mode=$(cat $MODE_FILE) mode=$(cat $MODE_FILE)
if [[ $mode == "server" ]]; then if [[ $mode == "server" ]]; then
curl --user-agent "authentik Healthcheck" -I http://localhost:9000/-/health/ready/ curl --user-agent "goauthentik.io lifecycle Healthcheck" -I http://localhost:9000/-/health/ready/
elif [[ $mode == "worker" ]]; then elif [[ $mode == "worker" ]]; then
celery -A authentik.root.celery inspect ping -d celery@$HOSTNAME celery -A authentik.root.celery inspect ping -d celery@$HOSTNAME
fi fi