internal: add internal healthchecking to prevent websocket errors
Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>
This commit is contained in:
parent
40055ef01b
commit
2aacb311bc
|
@ -54,7 +54,7 @@ func main() {
|
||||||
u, _ := url.Parse("http://localhost:8000")
|
u, _ := url.Parse("http://localhost:8000")
|
||||||
|
|
||||||
g := gounicorn.NewGoUnicorn()
|
g := gounicorn.NewGoUnicorn()
|
||||||
ws := web.NewWebServer()
|
ws := web.NewWebServer(g)
|
||||||
defer g.Kill()
|
defer g.Kill()
|
||||||
defer ws.Shutdown()
|
defer ws.Shutdown()
|
||||||
go web.RunMetricsServer()
|
go web.RunMetricsServer()
|
||||||
|
|
|
@ -1,10 +1,13 @@
|
||||||
package gounicorn
|
package gounicorn
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
"time"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
"goauthentik.io/internal/outpost/ak"
|
||||||
)
|
)
|
||||||
|
|
||||||
type GoUnicorn struct {
|
type GoUnicorn struct {
|
||||||
|
@ -12,6 +15,7 @@ type GoUnicorn struct {
|
||||||
p *exec.Cmd
|
p *exec.Cmd
|
||||||
started bool
|
started bool
|
||||||
killed bool
|
killed bool
|
||||||
|
alive bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGoUnicorn() *GoUnicorn {
|
func NewGoUnicorn() *GoUnicorn {
|
||||||
|
@ -20,6 +24,7 @@ func NewGoUnicorn() *GoUnicorn {
|
||||||
log: logger,
|
log: logger,
|
||||||
started: false,
|
started: false,
|
||||||
killed: false,
|
killed: false,
|
||||||
|
alive: false,
|
||||||
}
|
}
|
||||||
g.initCmd()
|
g.initCmd()
|
||||||
return g
|
return g
|
||||||
|
@ -35,6 +40,10 @@ func (g *GoUnicorn) initCmd() {
|
||||||
g.p.Stderr = os.Stderr
|
g.p.Stderr = os.Stderr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g *GoUnicorn) IsRunning() bool {
|
||||||
|
return g.alive
|
||||||
|
}
|
||||||
|
|
||||||
func (g *GoUnicorn) Start() error {
|
func (g *GoUnicorn) Start() error {
|
||||||
if g.killed {
|
if g.killed {
|
||||||
g.log.Debug("Not restarting gunicorn since we're killed")
|
g.log.Debug("Not restarting gunicorn since we're killed")
|
||||||
|
@ -44,9 +53,38 @@ func (g *GoUnicorn) Start() error {
|
||||||
g.initCmd()
|
g.initCmd()
|
||||||
}
|
}
|
||||||
g.started = true
|
g.started = true
|
||||||
|
go g.healthcheck()
|
||||||
return g.p.Run()
|
return g.p.Run()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (g *GoUnicorn) healthcheck() {
|
||||||
|
g.log.Debug("starting healthcheck")
|
||||||
|
h := &http.Client{
|
||||||
|
Transport: ak.NewUserAgentTransport("goauthentik.io go proxy healthcheck", http.DefaultTransport),
|
||||||
|
}
|
||||||
|
check := func() bool {
|
||||||
|
res, err := h.Get("http://localhost:8000/-/health/live/")
|
||||||
|
if err == nil && res.StatusCode == 204 {
|
||||||
|
g.alive = true
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default healthcheck is every 1 second on startup
|
||||||
|
// once we've been healthy once, increase to 30 seconds
|
||||||
|
for range time.Tick(time.Second) {
|
||||||
|
if check() {
|
||||||
|
g.log.Info("backend is alive, backing off with healthchecks")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
g.log.Debug("backend not alive yet")
|
||||||
|
}
|
||||||
|
for range time.Tick(30 * time.Second) {
|
||||||
|
check()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (g *GoUnicorn) Kill() {
|
func (g *GoUnicorn) Kill() {
|
||||||
g.killed = true
|
g.killed = true
|
||||||
err := g.p.Process.Kill()
|
err := g.p.Process.Kill()
|
||||||
|
|
|
@ -40,6 +40,10 @@ func (ws *WebServer) configureProxy() {
|
||||||
ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running"))
|
ws.proxyErrorHandler(rw, r, fmt.Errorf("proxy not running"))
|
||||||
})
|
})
|
||||||
ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
ws.m.PathPrefix("/").HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
||||||
|
if !ws.p.IsRunning() {
|
||||||
|
ws.proxyErrorHandler(rw, r, fmt.Errorf("authentik core not running yet"))
|
||||||
|
return
|
||||||
|
}
|
||||||
host := web.GetHost(r)
|
host := web.GetHost(r)
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
if ws.ProxyServer != nil {
|
if ws.ProxyServer != nil {
|
||||||
|
@ -59,8 +63,12 @@ func (ws *WebServer) configureProxy() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ws *WebServer) proxyErrorHandler(rw http.ResponseWriter, req *http.Request, err error) {
|
func (ws *WebServer) proxyErrorHandler(rw http.ResponseWriter, req *http.Request, err error) {
|
||||||
ws.log.WithError(err).Warning("proxy error")
|
ws.log.Warning(err.Error())
|
||||||
rw.WriteHeader(http.StatusBadGateway)
|
rw.WriteHeader(http.StatusBadGateway)
|
||||||
|
_, err = rw.Write([]byte("authentik starting..."))
|
||||||
|
if err != nil {
|
||||||
|
ws.log.WithError(err).Warning("failed to write error message")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ws *WebServer) proxyModifyResponse(r *http.Response) error {
|
func (ws *WebServer) proxyModifyResponse(r *http.Response) error {
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"github.com/pires/go-proxyproto"
|
"github.com/pires/go-proxyproto"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"goauthentik.io/internal/config"
|
"goauthentik.io/internal/config"
|
||||||
|
"goauthentik.io/internal/gounicorn"
|
||||||
"goauthentik.io/internal/outpost/proxyv2"
|
"goauthentik.io/internal/outpost/proxyv2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -27,9 +28,10 @@ type WebServer struct {
|
||||||
m *mux.Router
|
m *mux.Router
|
||||||
lh *mux.Router
|
lh *mux.Router
|
||||||
log *log.Entry
|
log *log.Entry
|
||||||
|
p *gounicorn.GoUnicorn
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewWebServer() *WebServer {
|
func NewWebServer(g *gounicorn.GoUnicorn) *WebServer {
|
||||||
l := log.WithField("logger", "authentik.g.web")
|
l := log.WithField("logger", "authentik.g.web")
|
||||||
mainHandler := mux.NewRouter()
|
mainHandler := mux.NewRouter()
|
||||||
if config.G.ErrorReporting.Enabled {
|
if config.G.ErrorReporting.Enabled {
|
||||||
|
@ -46,6 +48,7 @@ func NewWebServer() *WebServer {
|
||||||
m: mainHandler,
|
m: mainHandler,
|
||||||
lh: logginRouter,
|
lh: logginRouter,
|
||||||
log: l,
|
log: l,
|
||||||
|
p: g,
|
||||||
}
|
}
|
||||||
ws.configureStatic()
|
ws.configureStatic()
|
||||||
ws.configureProxy()
|
ws.configureProxy()
|
||||||
|
|
|
@ -49,7 +49,7 @@ elif [[ "$1" == "test" ]]; then
|
||||||
elif [[ "$1" == "healthcheck" ]]; then
|
elif [[ "$1" == "healthcheck" ]]; then
|
||||||
mode=$(cat $MODE_FILE)
|
mode=$(cat $MODE_FILE)
|
||||||
if [[ $mode == "server" ]]; then
|
if [[ $mode == "server" ]]; then
|
||||||
curl --user-agent "authentik Healthcheck" -I http://localhost:9000/-/health/ready/
|
curl --user-agent "goauthentik.io lifecycle Healthcheck" -I http://localhost:9000/-/health/ready/
|
||||||
elif [[ $mode == "worker" ]]; then
|
elif [[ $mode == "worker" ]]; then
|
||||||
celery -A authentik.root.celery inspect ping -d celery@$HOSTNAME
|
celery -A authentik.root.celery inspect ping -d celery@$HOSTNAME
|
||||||
fi
|
fi
|
||||||
|
|
Reference in a new issue