diff --git a/authentik/lib/default.yml b/authentik/lib/default.yml
index 6785ad32c..d0729a1f4 100644
--- a/authentik/lib/default.yml
+++ b/authentik/lib/default.yml
@@ -5,6 +5,10 @@ postgresql:
   user: authentik
   password: 'env://POSTGRES_PASSWORD'
 
+web:
+  listen: 0.0.0.0:9000
+  listen_tls: 0.0.0.0:9443
+
 redis:
   host: localhost
   password: ''
diff --git a/cmd/server/main.go b/cmd/server/main.go
index aa99020be..4f44e942b 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -10,7 +10,10 @@ import (
 )
 
 func main() {
+	log.SetLevel(log.DebugLevel)
 	config.DefaultConfig()
+	config.LoadConfig("./authentik/lib/default.yml")
+	config.LoadConfig("./local.env.yml")
 	config.ConfigureLogger()
 
 	rl := log.WithField("logger", "authentik.g")
diff --git a/go.mod b/go.mod
index 89962b90a..2735a40c7 100644
--- a/go.mod
+++ b/go.mod
@@ -6,5 +6,8 @@ require (
 	github.com/getsentry/sentry-go v0.10.0 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
 	github.com/sirupsen/logrus v1.8.1
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 878ca4f61..af24100ad 100644
--- a/go.sum
+++ b/go.sum
@@ -52,6 +52,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
@@ -100,6 +102,7 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -180,5 +183,8 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/config/config.go b/internal/config/config.go
index e182e658d..cdf66bccf 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,14 +1,20 @@
 package config
 
 import (
+	"io/ioutil"
+	"os"
+
+	"github.com/imdario/mergo"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 var G Config
 
 func DefaultConfig() {
 	G = Config{
-		Debug: true,
+		Debug: false,
 		Web: WebConfig{
 			Listen:    "localhost:9000",
 			ListenTLS: "localhost:9443",
@@ -16,15 +22,30 @@ func DefaultConfig() {
 		Paths: PathsConfig{
 			Media: "./media",
 		},
-		Log: LogConfig{
-			Level:  "trace",
-			Format: "json",
-		},
+		LogLevel: "info",
 	}
 }
 
+func LoadConfig(path string) error {
+	raw, err := ioutil.ReadFile(path)
+	if err != nil {
+		return errors.Wrap(err, "Failed to load config file")
+	}
+	rawExpanded := os.ExpandEnv(string(raw))
+	nc := Config{}
+	err = yaml.Unmarshal([]byte(rawExpanded), &nc)
+	if err != nil {
+		return errors.Wrap(err, "Failed to parse YAML")
+	}
+	if err := mergo.Merge(&G, nc, mergo.WithOverride); err != nil {
+		return errors.Wrap(err, "failed to overlay config")
+	}
+	log.WithField("path", path).Debug("Loaded config")
+	return nil
+}
+
 func ConfigureLogger() {
-	switch G.Log.Level {
+	switch G.LogLevel {
 	case "trace":
 		log.SetLevel(log.TraceLevel)
 	case "debug":
@@ -39,15 +60,14 @@ func ConfigureLogger() {
 		log.SetLevel(log.DebugLevel)
 	}
 
-	switch G.Log.Format {
-	case "json":
+	if G.Debug {
+		log.SetFormatter(&log.TextFormatter{})
+	} else {
 		log.SetFormatter(&log.JSONFormatter{
 			FieldMap: log.FieldMap{
 				log.FieldKeyMsg:  "event",
 				log.FieldKeyTime: "timestamp",
 			},
 		})
-	default:
-		log.SetFormatter(&log.TextFormatter{})
 	}
 }
diff --git a/internal/config/struct.go b/internal/config/struct.go
index d9540067f..683b6d5c7 100644
--- a/internal/config/struct.go
+++ b/internal/config/struct.go
@@ -1,22 +1,17 @@
 package config
 
 type Config struct {
-	Debug bool
-	Web   WebConfig
-	Paths PathsConfig
-	Log   LogConfig
+	Debug    bool        `yaml:"debug"`
+	Web      WebConfig   `yaml:"web"`
+	Paths    PathsConfig `yaml:"paths"`
+	LogLevel string      `yaml:"log_level"`
 }
 
 type WebConfig struct {
-	Listen    string
-	ListenTLS string
+	Listen    string `yaml:"listen"`
+	ListenTLS string `yaml:"listen_tls"`
 }
 
 type PathsConfig struct {
-	Media string
-}
-
-type LogConfig struct {
-	Level  string
-	Format string
+	Media string `yaml:"media"`
 }