diff --git a/cmd/ldap/server.go b/cmd/ldap/server.go
index 7b189559e..724b86d2b 100644
--- a/cmd/ldap/server.go
+++ b/cmd/ldap/server.go
@@ -8,6 +8,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"goauthentik.io/internal/common"
+	"goauthentik.io/internal/config"
 	"goauthentik.io/internal/debug"
 	"goauthentik.io/internal/outpost/ak"
 	"goauthentik.io/internal/outpost/ldap"
@@ -30,14 +31,14 @@ func main() {
 		DisableHTMLEscape: true,
 	})
 	go debug.EnableDebugServer()
-	akURL, found := os.LookupEnv("AUTHENTIK_HOST")
-	if !found {
+	akURL := config.Get().AuthentikHost
+	if akURL == "" {
 		fmt.Println("env AUTHENTIK_HOST not set!")
 		fmt.Println(helpMessage)
 		os.Exit(1)
 	}
-	akToken, found := os.LookupEnv("AUTHENTIK_TOKEN")
-	if !found {
+	akToken := config.Get().AuthentikToken
+	if akToken == "" {
 		fmt.Println("env AUTHENTIK_TOKEN not set!")
 		fmt.Println(helpMessage)
 		os.Exit(1)
diff --git a/cmd/proxy/server.go b/cmd/proxy/server.go
index cd0074405..49ef5fa95 100644
--- a/cmd/proxy/server.go
+++ b/cmd/proxy/server.go
@@ -8,6 +8,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"goauthentik.io/internal/common"
+	"goauthentik.io/internal/config"
 	"goauthentik.io/internal/debug"
 	"goauthentik.io/internal/outpost/ak"
 	"goauthentik.io/internal/outpost/proxyv2"
@@ -33,14 +34,14 @@ func main() {
 		DisableHTMLEscape: true,
 	})
 	go debug.EnableDebugServer()
-	akURL, found := os.LookupEnv("AUTHENTIK_HOST")
-	if !found {
+	akURL := config.Get().AuthentikHost
+	if akURL == "" {
 		fmt.Println("env AUTHENTIK_HOST not set!")
 		fmt.Println(helpMessage)
 		os.Exit(1)
 	}
-	akToken, found := os.LookupEnv("AUTHENTIK_TOKEN")
-	if !found {
+	akToken := config.Get().AuthentikToken
+	if akToken == "" {
 		fmt.Println("env AUTHENTIK_TOKEN not set!")
 		fmt.Println(helpMessage)
 		os.Exit(1)
diff --git a/internal/config/struct.go b/internal/config/struct.go
index 9a087376a..f8d6e3b28 100644
--- a/internal/config/struct.go
+++ b/internal/config/struct.go
@@ -1,14 +1,23 @@
 package config
 
 type Config struct {
-	Debug          bool                 `yaml:"debug" env:"AUTHENTIK_DEBUG"`
+	// Core specific config
 	SecretKey      string               `yaml:"secret_key" env:"AUTHENTIK_SECRET_KEY"`
-	Listen         ListenConfig         `yaml:"listen"`
 	Paths          PathsConfig          `yaml:"paths"`
 	LogLevel       string               `yaml:"log_level" env:"AUTHENTIK_LOG_LEVEL"`
 	ErrorReporting ErrorReportingConfig `yaml:"error_reporting"`
 	Redis          RedisConfig          `yaml:"redis"`
-	Outposts       OutpostConfig        `yaml:"outposts" `
+	Outposts       OutpostConfig        `yaml:"outposts"`
+
+	// Config for both core and outposts
+	Debug  bool         `yaml:"debug" env:"AUTHENTIK_DEBUG"`
+	Listen ListenConfig `yaml:"listen"`
+
+	// Outpost specific config
+	// These are only relevant for proxy/ldap outposts, and cannot be set via YAML
+	// They are loaded via this config loader to support file:// schemas
+	AuthentikHost  string `env:"AUTHENTIK_HOST"`
+	AuthentikToken string `env:"AUTHENTIK_TOKEN"`
 }
 
 type RedisConfig struct {
diff --git a/internal/debug/debug.go b/internal/debug/debug.go
index 199367445..353e445c1 100644
--- a/internal/debug/debug.go
+++ b/internal/debug/debug.go
@@ -3,8 +3,6 @@ package debug
 import (
 	"net/http"
 	"net/http/pprof"
-	"os"
-	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"goauthentik.io/internal/config"
@@ -12,7 +10,7 @@ import (
 
 func EnableDebugServer() {
 	l := log.WithField("logger", "authentik.go_debugger")
-	if deb := os.Getenv("AUTHENTIK_DEBUG"); strings.ToLower(deb) != "true" {
+	if !config.Get().Debug {
 		l.Info("not enabling debug server, set `AUTHENTIK_DEBUG` to `true` to enable it.")
 		return
 	}