replace ioutil

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

authentik/lib/default.yml

@@ -11,7 +11,11 @@ postgresql:
 listen:
   listen_http: 0.0.0.0:9000
   listen_https: 0.0.0.0:9443
+  listen_ldap: 0.0.0.0:3389
+  listen_ldaps: 0.0.0.0:6636
+  listen_radius: 0.0.0.0:1812
   listen_metrics: 0.0.0.0:9300
+  listen_debug: 0.0.0.0:9900
 
 redis:
   host: localhost
@@ -25,6 +29,9 @@ redis:
   cache_timeout_policies: 300
   cache_timeout_reputation: 300
 
+paths:
+  media: ./media
+
 debug: false
 
 log_level: info

cmd/server/server.go

@@ -4,11 +4,15 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"github.com/getsentry/sentry-go"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+
 	"goauthentik.io/internal/common"
 	"goauthentik.io/internal/config"
 	"goauthentik.io/internal/constants"
@@ -70,6 +74,21 @@ var rootCmd = &cobra.Command{
 			l.Info("shutting down gunicorn")
 			g.Kill()
 		}()
+
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, syscall.SIGHUP, syscall.SIGUSR2)
+		go func() {
+			sig := <-c
+			if sig == syscall.SIGHUP {
+				log.Info("SIGHUP received, forwarding to gunicorn")
+				g.Reload()
+			}
+			if sig == syscall.SIGUSR2 {
+				log.Info("SIGUSR2 received, restarting gunicorn")
+				g.Restart()
+			}
+		}()
+
 		ws := web.NewWebServer(g)
 		g.HealthyCallback = func() {
 			if !config.Get().Outposts.DisableEmbeddedOutpost {
@@ -78,6 +97,17 @@ var rootCmd = &cobra.Command{
 		}
 		go web.RunMetricsServer()
 		go attemptStartBackend(g)
+
+		w, err := config.WatchChanges(func() {
+			g.Restart()
+		})
+		if err != nil {
+			l.WithError(err).Warning("failed to start watching for configuration changes, no automatic update will be done")
+		}
+		if w != nil {
+			defer w.Close()
+		}
+
 		ws.Start()
 		<-ex
 		running = false
@@ -92,8 +122,24 @@ func attemptStartBackend(g *gounicorn.GoUnicorn) {
 		if !running {
 			return
 		}
+		g.Kill()
+		log.WithField("logger", "authentik.router").Info("starting gunicorn")
 		err := g.Start()
-		log.WithField("logger", "authentik.router").WithError(err).Warning("gunicorn process died, restarting")
+		if err != nil {
+			log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn failed to start, restarting")
+			continue
+		}
+		failedChecks := 0
+		for range time.Tick(30 * time.Second) {
+			if !g.IsRunning() {
+				log.WithField("logger", "authentik.router").Warningf("gunicorn process failed healthcheck %d times", failedChecks)
+				failedChecks += 1
+			}
+			if failedChecks >= 3 {
+				log.WithField("logger", "authentik.router").WithError(err).Error("gunicorn process failed healthcheck three times, restarting")
+				break
+			}
+		}
 	}
 }
 

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-http-utils/fresh v0.0.0-20161124030543-7231e26a4b27 // indirect
 	github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a // indirect

go.sum

@@ -66,6 +66,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/garyburd/redigo v1.6.4 h1:LFu2R3+ZOPgSMWMOL+saa/zXRjw0ID2G8FepO53BGlg=
 github.com/garyburd/redigo v1.6.4/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
 github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
@@ -486,6 +488,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/config/config.go

@@ -5,46 +5,134 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
+	"time"
 
 	env "github.com/Netflix/go-env"
+	"github.com/fsnotify/fsnotify"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
 
 var cfg *Config
 
+func getConfigPaths() []string {
+	configPaths := []string{"./authentik/lib/default.yml", "/etc/authentik/config.yml", ""}
+	globConfigPaths, _ := filepath.Glob("/etc/authentik/config.d/*.yml")
+	configPaths = append(configPaths, globConfigPaths...)
+
+	environment := "local"
+	if v, ok := os.LookupEnv("AUTHENTIK_ENV"); ok {
+		environment = v
+	}
+
+	computedConfigPaths := []string{}
+
+	for _, path := range configPaths {
+		path, err := filepath.Abs(path)
+		if err != nil {
+			continue
+		}
+		if stat, err := os.Stat(path); err == nil {
+			if !stat.IsDir() {
+				computedConfigPaths = append(computedConfigPaths, path)
+			} else {
+				envPaths := []string{
+					filepath.Join(path, environment+".yml"),
+					filepath.Join(path, environment+".env.yml"),
+				}
+				for _, envPath := range envPaths {
+					if stat, err = os.Stat(envPath); err == nil && !stat.IsDir() {
+						computedConfigPaths = append(computedConfigPaths, envPath)
+					}
+				}
+			}
+		}
+	}
+
+	return computedConfigPaths
+}
+
+func WatchChanges(callback func()) (*fsnotify.Watcher, error) {
+	configPaths := getConfigPaths()
+
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	// Start listening for events
+	go func() {
+		reload := false
+		for {
+			select {
+			// Read from errors
+			case err, ok := <-w.Errors:
+				if !ok {
+					// Channel was closed (i.e. Watcher.Close() was called)
+					return
+				}
+				log.WithError(err).Warning("failed to watch for file changes")
+			// Read from events
+			case ev, ok := <-w.Events:
+				if !ok {
+					// Channel was closed (i.e. Watcher.Close() was called)
+					return
+				}
+
+				// Ignores files we're not interested in
+				// We get the whole list again in case a file in a directory we're watching was created
+				// and that file is of interest
+				for _, f := range getConfigPaths() {
+					if f == ev.Name {
+						reload = true
+						log.Debugf("%s file change, setting configuration to be reloaded", ev.Name)
+					}
+				}
+			default:
+				if reload {
+					log.Infof("configuration files changed, reloading configuration")
+					Reload()
+					callback()
+					// Cooldown after configuration reloading
+					time.Sleep(14 * time.Second)
+				}
+				// Otherwise `select` fires all of the time
+				time.Sleep(1 * time.Second)
+				reload = false
+			}
+		}
+	}()
+
+	// Register the files to be watch
+	for _, p := range configPaths {
+		// Actually, watch the directory, not the files, as that's how fsnotify works
+		// We assume we're only getting valid files as getConfigPaths already handles all the necessary checks
+		err = w.Add(filepath.Dir(p))
+		log.Warningf("watching for changes for file %s in dir %s", p, filepath.Dir(p))
+		if err != nil {
+			return w, err
+		}
+	}
+
+	return w, nil
+}
+
 func Get() *Config {
 	if cfg == nil {
-		c := defaultConfig()
-		c.Setup("./authentik/lib/default.yml", "/etc/authentik/config.yml", "./local.env.yml")
+		c := &Config{}
+		c.Setup(getConfigPaths()...)
 		cfg = c
 	}
 	return cfg
 }
 
-func defaultConfig() *Config {
-	return &Config{
-		Debug: false,
-		Listen: ListenConfig{
-			HTTP:    "0.0.0.0:9000",
-			HTTPS:   "0.0.0.0:9443",
-			LDAP:    "0.0.0.0:3389",
-			LDAPS:   "0.0.0.0:6636",
-			Radius:  "0.0.0.0:1812",
-			Metrics: "0.0.0.0:9300",
-			Debug:   "0.0.0.0:9900",
-		},
-		Paths: PathsConfig{
-			Media: "./media",
-		},
-		LogLevel: "info",
-		ErrorReporting: ErrorReportingConfig{
-			Enabled:    false,
-			SampleRate: 1,
-		},
-	}
+func Reload() {
+	c := &Config{}
+	c.Setup(getConfigPaths()...)
+	cfg = c
 }
 
 func (c *Config) Setup(paths ...string) {

internal/gounicorn/gounicorn.go

@@ -1,15 +1,20 @@
 package gounicorn
 
 import (
+	"fmt"
 	"net/http"
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
 	log "github.com/sirupsen/logrus"
+
 	"goauthentik.io/internal/config"
+	"goauthentik.io/internal/utils"
 	"goauthentik.io/internal/utils/web"
 )
 
@@ -18,6 +23,7 @@ type GoUnicorn struct {
 
 	log     *log.Entry
 	p       *exec.Cmd
+	pidFile *string
 	started bool
 	killed  bool
 	alive   bool
@@ -27,6 +33,7 @@ func New() *GoUnicorn {
 	logger := log.WithField("logger", "authentik.router.unicorn")
 	g := &GoUnicorn{
 		log:             logger,
+		pidFile:         nil,
 		started:         false,
 		killed:          false,
 		alive:           false,
@@ -37,8 +44,13 @@ func New() *GoUnicorn {
 }
 
 func (g *GoUnicorn) initCmd() {
+	pidFile, _ := os.CreateTemp("", "authentik-gunicorn.*.pid")
+	g.pidFile = func() *string { s := pidFile.Name(); return &s }()
 	command := "gunicorn"
 	args := []string{"-c", "./lifecycle/gunicorn.conf.py", "authentik.root.asgi:application"}
+	if g.pidFile != nil {
+		args = append(args, "--pid", *g.pidFile)
+	}
 	if config.Get().Debug {
 		command = "./manage.py"
 		args = []string{"runserver"}
@@ -55,16 +67,13 @@ func (g *GoUnicorn) IsRunning() bool {
 }
 
 func (g *GoUnicorn) Start() error {
-	if g.killed {
-		g.log.Debug("Not restarting gunicorn since we're shutdown")
-		return nil
-	}
 	if g.started {
 		g.initCmd()
 	}
+	g.killed = false
 	g.started = true
 	go g.healthcheck()
-	return g.p.Run()
+	return g.p.Start()
 }
 
 func (g *GoUnicorn) healthcheck() {
@@ -96,8 +105,77 @@ func (g *GoUnicorn) healthcheck() {
 	}
 }
 
+func (g *GoUnicorn) Reload() {
+	g.log.WithField("method", "reload").Info("reloading gunicorn")
+	err := g.p.Process.Signal(syscall.SIGHUP)
+	if err != nil {
+		g.log.WithError(err).Warning("failed to reload gunicorn")
+	}
+}
+
+func (g *GoUnicorn) Restart() {
+	g.log.WithField("method", "restart").Info("restart gunicorn")
+	if g.pidFile == nil {
+		g.log.Warning("pidfile is non existent, cannot restart")
+		return
+	}
+
+	err := g.p.Process.Signal(syscall.SIGUSR2)
+	if err != nil {
+		g.log.WithError(err).Warning("failed to restart gunicorn")
+		return
+	}
+
+	newPidFile := fmt.Sprintf("%s.2", *g.pidFile)
+
+	// Wait for the new PID file to be created
+	for range time.Tick(1 * time.Second) {
+		_, err = os.Stat(newPidFile)
+		if err == nil || !os.IsNotExist(err) {
+			break
+		}
+		g.log.Debugf("waiting for new gunicorn pidfile to appear at %s", newPidFile)
+	}
+	if err != nil {
+		g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
+		return
+	}
+
+	newPidB, err := os.ReadFile(newPidFile)
+	if err != nil {
+		g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
+		return
+	}
+	newPidS := strings.TrimSpace(string(newPidB[:]))
+	newPid, err := strconv.Atoi(newPidS)
+	if err != nil {
+		g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
+		return
+	}
+	g.log.Warningf("new gunicorn PID is %d", newPid)
+
+	newProcess, err := utils.FindProcess(newPid)
+	if newProcess == nil || err != nil {
+		g.log.WithError(err).Warning("failed to find the new gunicorn process, aborting")
+		return
+	}
+
+	// The new process has started, let's gracefully kill the old one
+	g.log.Warningf("killing old gunicorn")
+	err = g.p.Process.Signal(syscall.SIGTERM)
+	if err != nil {
+		g.log.Warning("failed to kill old instance of gunicorn")
+	}
+
+	g.p.Process = newProcess
+
+	// No need to close any files and the .2 pid file is deleted by Gunicorn
+}
+
 func (g *GoUnicorn) Kill() {
-	g.killed = true
+	if !g.started {
+		return
+	}
 	var err error
 	if runtime.GOOS == "darwin" {
 		g.log.WithField("method", "kill").Warning("stopping gunicorn")
@@ -109,4 +187,8 @@ func (g *GoUnicorn) Kill() {
 	if err != nil {
 		g.log.WithError(err).Warning("failed to stop gunicorn")
 	}
+	if g.pidFile != nil {
+		os.Remove(*g.pidFile)
+	}
+	g.killed = true
 }

internal/utils/process.go

@@ -0,0 +1,39 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"syscall"
+)
+
+func FindProcess(pid int) (*os.Process, error) {
+	if pid <= 0 {
+		return nil, fmt.Errorf("invalid pid %v", pid)
+	}
+	// The error doesn't mean anything on Unix systems, let's just check manually
+	// that the new gunicorn master has properly started
+	// https://github.com/golang/go/issues/34396
+	proc, err := os.FindProcess(int(pid))
+	if err != nil {
+		return nil, err
+	}
+	err = proc.Signal(syscall.Signal(0))
+	if err == nil {
+		return proc, nil
+	}
+	if errors.Is(err, os.ErrProcessDone) {
+		return nil, nil
+	}
+	errno, ok := err.(syscall.Errno)
+	if !ok {
+		return nil, err
+	}
+	switch errno {
+	case syscall.ESRCH:
+		return nil, nil
+	case syscall.EPERM:
+		return proc, nil
+	}
+	return nil, err
+}