Merge branch '30-application-security-gateway' into 'master'

Resolve "Application Security Gateway (Reverse Proxy)"

Closes #30

See merge request BeryJu.org/passbook!17
This commit is contained in:
Jens Langhammer 2019-03-21 15:41:34 +00:00
commit 457375287c
29 changed files with 572 additions and 10 deletions

View file

@ -53,3 +53,4 @@ values =
[bumpversion:file:passbook/otp/__init__.py]
[bumpversion:file:passbook/app_gw/__init__.py]

View file

@ -16,6 +16,8 @@ redis: localhost/0
# Error reporting, sends stacktrace to sentry.services.beryju.org
error_report_enabled: true
primary_domain: passbook.local
passbook:
sign_up:
# Enables signup, created users are stored in internal Database and created in LDAP if ldap.create_users is true

View file

@ -47,6 +47,7 @@ data:
secret_key: {{ randAlphaNum 50 }}
{{- end }}
primary_domain: {{ .Values.primary_domain }}
domains:
{{- range .Values.ingress.hosts }}
- {{ . | quote }}

View file

@ -36,7 +36,7 @@
<tbody>
{% for property_mapping in object_list %}
<tr>
<td>{{ property_mapping.name }} ({{ property_mapping.slug }})</td>
<td>{{ property_mapping.name }}</td>
<td>{{ property_mapping|verbose_name }}</td>
<td>
<a class="btn btn-default btn-sm"

BIN
passbook/app_gw/.DS_Store vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,2 @@
"""passbook Application Security Gateway Header"""
__version__ = '0.1.23-beta'

5
passbook/app_gw/admin.py Normal file
View file

@ -0,0 +1,5 @@
"""passbook Application Security Gateway model admin"""
from passbook.lib.admin import admin_autoregister
admin_autoregister('passbook_app_gw')

16
passbook/app_gw/apps.py Normal file
View file

@ -0,0 +1,16 @@
"""passbook Application Security Gateway app"""
from importlib import import_module
from django.apps import AppConfig
class PassbookApplicationApplicationGatewayConfig(AppConfig):
"""passbook app_gw app"""
name = 'passbook.app_gw'
label = 'passbook_app_gw'
verbose_name = 'passbook Application Security Gateway'
mountpoint = 'app_gw/'
def ready(self):
import_module('passbook.app_gw.signals')

56
passbook/app_gw/forms.py Normal file
View file

@ -0,0 +1,56 @@
"""passbook Application Security Gateway Forms"""
from django import forms
from django.contrib.admin.widgets import FilteredSelectMultiple
from django.forms import ValidationError
from django.utils.translation import gettext as _
from passbook.app_gw.models import ApplicationGatewayProvider, RewriteRule
from passbook.lib.fields import DynamicArrayField
class ApplicationGatewayProviderForm(forms.ModelForm):
"""Security Gateway Provider form"""
def clean_server_name(self):
"""Check if server_name is in DB already, since
Postgres ArrayField doesn't suppport keys."""
current = self.cleaned_data.get('server_name')
if ApplicationGatewayProvider.objects \
.filter(server_name__overlap=current) \
.exclude(pk=self.instance.pk).exists():
raise ValidationError("Server Name already in use.")
return current
class Meta:
model = ApplicationGatewayProvider
fields = ['server_name', 'upstream', 'enabled', 'authentication_header',
'default_content_type', 'upstream_ssl_verification', 'property_mappings']
widgets = {
'authentication_header': forms.TextInput(),
'default_content_type': forms.TextInput(),
'property_mappings': FilteredSelectMultiple(_('Property Mappings'), False)
}
field_classes = {
'server_name': DynamicArrayField,
'upstream': DynamicArrayField
}
labels = {
'upstream_ssl_verification': _('Verify upstream SSL Certificates?'),
'property_mappings': _('Rewrite Rules')
}
class RewriteRuleForm(forms.ModelForm):
"""Rewrite Rule Form"""
class Meta:
model = RewriteRule
fields = ['name', 'match', 'halt', 'replacement', 'redirect', 'conditions']
widgets = {
'name': forms.TextInput(),
'match': forms.TextInput(attrs={'data-is-monospace': True}),
'replacement': forms.TextInput(attrs={'data-is-monospace': True}),
'conditions': FilteredSelectMultiple(_('Conditions'), False)
}

View file

@ -0,0 +1,227 @@
"""passbook app_gw middleware"""
import mimetypes
from logging import getLogger
from urllib.parse import urlparse
import certifi
import urllib3
from django.core.cache import cache
from django.utils.http import urlencode
from revproxy.exceptions import InvalidUpstream
from revproxy.response import get_django_response
from revproxy.utils import encode_items, normalize_request_headers
from passbook.app_gw.models import ApplicationGatewayProvider
from passbook.app_gw.rewrite import Rewriter
from passbook.core.models import Application
from passbook.core.policies import PolicyEngine
IGNORED_HOSTNAMES_KEY = 'passbook_app_gw_ignored'
LOGGER = getLogger(__name__)
QUOTE_SAFE = r'<.;>\(}*+|~=-$/_:^@)[{]&\'!,"`'
ERRORS_MESSAGES = {
'upstream-no-scheme': ("Upstream URL scheme must be either "
"'http' or 'https' (%s).")
}
# pylint: disable=too-many-instance-attributes
class ApplicationGatewayMiddleware:
"""Check if request should be proxied or handeled normally"""
ignored_hosts = []
request = None
app_gw = None
http = None
http_no_verify = None
host_header = ''
_parsed_url = None
_request_headers = None
def __init__(self, get_response):
self.get_response = get_response
self.ignored_hosts = cache.get(IGNORED_HOSTNAMES_KEY, [])
self.http_no_verify = urllib3.PoolManager()
self.http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
def precheck(self, request):
"""Check if a request should be proxied or forwarded to passbook"""
# Check if hostname is in cached list of ignored hostnames
# This saves us having to query the database on each request
self.host_header = request.META.get('HTTP_HOST')
if self.host_header in self.ignored_hosts:
LOGGER.debug("%s is ignored", self.host_header)
return True, None
# Look through all ApplicationGatewayProviders and check hostnames
matches = ApplicationGatewayProvider.objects.filter(
server_name__contains=[self.host_header],
enabled=True)
if not matches.exists():
# Mo matching Providers found, add host header to ignored list
self.ignored_hosts.append(self.host_header)
cache.set(IGNORED_HOSTNAMES_KEY, self.ignored_hosts)
LOGGER.debug("Ignoring %s", self.host_header)
return True, None
# At this point we're certain there's a matching ApplicationGateway
if len(matches) > 1:
# TODO This should never happen
raise ValueError
app_gw = matches.first()
try:
# Check if ApplicationGateway is associcaited with application
getattr(app_gw, 'application')
return False, app_gw
except Application.DoesNotExist:
LOGGER.debug("ApplicationGateway not associated with Application")
return True, None
return True, None
def __call__(self, request):
forward, self.app_gw = self.precheck(request)
if forward:
return self.get_response(request)
self.request = request
return self.dispatch(request)
def get_upstream(self):
"""Get upstream as parsed url"""
# TODO: How to choose upstream?
upstream = self.app_gw.upstream[0]
if not getattr(self, '_parsed_url', None):
self._parsed_url = urlparse(upstream)
if self._parsed_url.scheme not in ('http', 'https'):
raise InvalidUpstream(ERRORS_MESSAGES['upstream-no-scheme'] %
upstream)
return upstream
def _format_path_to_redirect(self, request):
LOGGER.debug("Path before: %s", request.get_full_path())
rewriter = Rewriter(self.app_gw, request)
after = rewriter.build()
LOGGER.debug("Path after: %s", after)
return after
def get_proxy_request_headers(self, request):
"""Get normalized headers for the upstream
Gets all headers from the original request and normalizes them.
Normalization occurs by removing the prefix ``HTTP_`` and
replacing and ``_`` by ``-``. Example: ``HTTP_ACCEPT_ENCODING``
becames ``Accept-Encoding``.
.. versionadded:: 0.9.1
:param request: The original HTTPRequest instance
:returns: Normalized headers for the upstream
"""
return normalize_request_headers(request)
def get_request_headers(self):
"""Return request headers that will be sent to upstream.
The header REMOTE_USER is set to the current user
if AuthenticationMiddleware is enabled and
the view's add_remote_user property is True.
.. versionadded:: 0.9.8
"""
request_headers = self.get_proxy_request_headers(self.request)
request_headers[self.app_gw.authentication_header] = self.request.user.get_username()
LOGGER.info("%s set", self.app_gw.authentication_header)
return request_headers
def check_permission(self):
"""Check if user is authenticated and has permission to access app"""
if not hasattr(self.request, 'user'):
return False
if not self.request.user.is_authenticated:
return False
policy_engine = PolicyEngine(self.app_gw.application.policies.all())
policy_engine.for_user(self.request.user).with_request(self.request).build()
passing, _messages = policy_engine.result
return passing
def get_encoded_query_params(self):
"""Return encoded query params to be used in proxied request"""
get_data = encode_items(self.request.GET.lists())
return urlencode(get_data)
def _created_proxy_response(self, request, path):
request_payload = request.body
LOGGER.debug("Request headers: %s", self._request_headers)
request_url = self.get_upstream() + path
LOGGER.debug("Request URL: %s", request_url)
if request.GET:
request_url += '?' + self.get_encoded_query_params()
LOGGER.debug("Request URL: %s", request_url)
http = self.http
if not self.app_gw.upstream_ssl_verification:
http = self.http_no_verify
try:
proxy_response = http.urlopen(request.method,
request_url,
redirect=False,
retries=None,
headers=self._request_headers,
body=request_payload,
decode_content=False,
preload_content=False)
LOGGER.debug("Proxy response header: %s",
proxy_response.getheaders())
except urllib3.exceptions.HTTPError as error:
LOGGER.exception(error)
raise
return proxy_response
def _replace_host_on_redirect_location(self, request, proxy_response):
location = proxy_response.headers.get('Location')
if location:
if request.is_secure():
scheme = 'https://'
else:
scheme = 'http://'
request_host = scheme + self.host_header
upstream_host_http = 'http://' + self._parsed_url.netloc
upstream_host_https = 'https://' + self._parsed_url.netloc
location = location.replace(upstream_host_http, request_host)
location = location.replace(upstream_host_https, request_host)
proxy_response.headers['Location'] = location
LOGGER.debug("Proxy response LOCATION: %s",
proxy_response.headers['Location'])
def _set_content_type(self, request, proxy_response):
content_type = proxy_response.headers.get('Content-Type')
if not content_type:
content_type = (mimetypes.guess_type(request.path)[0] or
self.app_gw.default_content_type)
proxy_response.headers['Content-Type'] = content_type
LOGGER.debug("Proxy response CONTENT-TYPE: %s",
proxy_response.headers['Content-Type'])
def dispatch(self, request):
"""Build proxied request and pass to upstream"""
# if not self.check_permission():
# to_url = 'https://%s/?next=%s' % (CONFIG.get('domains')[0], request.get_full_path())
# return RedirectView.as_view(url=to_url)(request)
self._request_headers = self.get_request_headers()
path = self._format_path_to_redirect(request)
proxy_response = self._created_proxy_response(request, path)
self._replace_host_on_redirect_location(request, proxy_response)
self._set_content_type(request, proxy_response)
response = get_django_response(proxy_response, strict_cookies=False)
LOGGER.debug("RESPONSE RETURNED: %s", response)
return response

BIN
passbook/app_gw/migrations/.DS_Store vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,50 @@
# Generated by Django 2.1.7 on 2019-03-20 21:38
import django.contrib.postgres.fields
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('passbook_core', '0020_groupmembershippolicy'),
]
operations = [
migrations.CreateModel(
name='ApplicationGatewayProvider',
fields=[
('provider_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_core.Provider')),
('server_name', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), size=None)),
('upstream', django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), size=None)),
('enabled', models.BooleanField(default=True)),
('authentication_header', models.TextField(default='X-Remote-User')),
('default_content_type', models.TextField(default='application/octet-stream')),
('upstream_ssl_verification', models.BooleanField(default=True)),
],
options={
'verbose_name': 'Application Gateway Provider',
'verbose_name_plural': 'Application Gateway Providers',
},
bases=('passbook_core.provider',),
),
migrations.CreateModel(
name='RewriteRule',
fields=[
('propertymapping_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_core.PropertyMapping')),
('match', models.TextField()),
('halt', models.BooleanField(default=False)),
('replacement', models.TextField()),
('redirect', models.CharField(choices=[('internal', 'Internal'), (301, 'Moved Permanently'), (302, 'Found')], max_length=50)),
('conditions', models.ManyToManyField(to='passbook_core.Policy')),
],
options={
'verbose_name': 'Rewrite Rule',
'verbose_name_plural': 'Rewrite Rules',
},
bases=('passbook_core.propertymapping',),
),
]

View file

@ -0,0 +1,18 @@
# Generated by Django 2.1.7 on 2019-03-21 15:21
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('passbook_app_gw', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='rewriterule',
name='conditions',
field=models.ManyToManyField(blank=True, to='passbook_core.Policy'),
),
]

View file

74
passbook/app_gw/models.py Normal file
View file

@ -0,0 +1,74 @@
"""passbook app_gw models"""
import re
from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.utils.translation import gettext as _
from passbook.core.models import Policy, PropertyMapping, Provider
class ApplicationGatewayProvider(Provider):
"""Virtual server which proxies requests to any hostname in server_name to upstream"""
server_name = ArrayField(models.TextField())
upstream = ArrayField(models.TextField())
enabled = models.BooleanField(default=True)
authentication_header = models.TextField(default='X-Remote-User')
default_content_type = models.TextField(default='application/octet-stream')
upstream_ssl_verification = models.BooleanField(default=True)
form = 'passbook.app_gw.forms.ApplicationGatewayProviderForm'
@property
def name(self):
"""since this model has no name property, return a joined list of server_names as name"""
return ', '.join(self.server_name)
def __str__(self):
return "Application Gateway %s" % ', '.join(self.server_name)
class Meta:
verbose_name = _('Application Gateway Provider')
verbose_name_plural = _('Application Gateway Providers')
class RewriteRule(PropertyMapping):
"""Rewrite requests matching `match` with `replacement`, if all polcies in `conditions` apply"""
REDIRECT_INTERNAL = 'internal'
REDIRECT_PERMANENT = 301
REDIRECT_FOUND = 302
REDIRECTS = (
(REDIRECT_INTERNAL, _('Internal')),
(REDIRECT_PERMANENT, _('Moved Permanently')),
(REDIRECT_FOUND, _('Found')),
)
match = models.TextField()
halt = models.BooleanField(default=False)
conditions = models.ManyToManyField(Policy, blank=True)
replacement = models.TextField() # python formatted strings, use {match.1}
redirect = models.CharField(max_length=50, choices=REDIRECTS)
form = 'passbook.app_gw.forms.RewriteRuleForm'
_matcher = None
@property
def compiled_matcher(self):
"""Cache the compiled regex in memory"""
if not self._matcher:
self._matcher = re.compile(self.match)
return self._matcher
def __str__(self):
return "Rewrite Rule %s" % self.name
class Meta:
verbose_name = _('Rewrite Rule')
verbose_name_plural = _('Rewrite Rules')

View file

@ -0,0 +1,2 @@
django-revproxy
urllib3[secure]

View file

@ -0,0 +1,38 @@
"""passbook app_gw rewriter"""
from passbook.app_gw.models import RewriteRule
class Context:
"""Empty class which we dynamically add attributes to"""
class Rewriter:
"""Apply rewrites"""
__application = None
__request = None
def __init__(self, application, request):
self.__application = application
self.__request = request
def __build_context(self, matches):
"""Build object with .0, .1, etc as groups and give access to request"""
context = Context()
for index, group_match in enumerate(matches.groups()):
setattr(context, "g%d" % (index + 1), group_match)
setattr(context, 'request', self.__request)
return context
def build(self):
"""Run all rules over path and return final path"""
path = self.__request.get_full_path()
for rule in RewriteRule.objects.filter(provider__in=[self.__application]):
matches = rule.compiled_matcher.search(path)
if not matches:
continue
replace_context = self.__build_context(matches)
path = rule.replacement.format(context=replace_context)
if rule.halt:
return path
return path

View file

@ -0,0 +1,5 @@
"""Application Security Gateway settings"""
# INSTALLED_APPS = [
# 'revproxy'
# ]

View file

@ -0,0 +1,20 @@
"""passbook app_gw cache clean signals"""
from logging import getLogger
from django.core.cache import cache
from django.db.models.signals import post_save
from django.dispatch import receiver
from passbook.app_gw.middleware import IGNORED_HOSTNAMES_KEY
from passbook.app_gw.models import ApplicationGatewayProvider
LOGGER = getLogger(__name__)
@receiver(post_save)
# pylint: disable=unused-argument
def invalidate_app_gw_cache(sender, instance, **kwargs):
"""Invalidate Policy cache when app_gw is updated"""
if isinstance(instance, ApplicationGatewayProvider):
LOGGER.debug("Invalidating cache for ignored hostnames")
cache.delete(IGNORED_HOSTNAMES_KEY)

2
passbook/app_gw/urls.py Normal file
View file

@ -0,0 +1,2 @@
"""passbook app_gw urls"""
urlpatterns = []

View file

@ -7,7 +7,7 @@ from passbook.core.models import (DebugPolicy, FieldMatcherPolicy,
GroupMembershipPolicy, PasswordPolicy,
WebhookPolicy)
GENERAL_FIELDS = ['name', 'action', 'negate', 'order', ]
GENERAL_FIELDS = ['name', 'action', 'negate', 'order', 'timeout']
class FieldMatcherPolicyForm(forms.ModelForm):
"""FieldMatcherPolicy Form"""

View file

@ -0,0 +1,18 @@
# Generated by Django 2.1.7 on 2019-03-21 12:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('passbook_core', '0020_groupmembershippolicy'),
]
operations = [
migrations.AddField(
model_name='policy',
name='timeout',
field=models.IntegerField(default=30),
),
]

View file

@ -220,6 +220,7 @@ class Policy(UUIDModel, CreatedUpdatedModel):
action = models.CharField(max_length=20, choices=ACTIONS)
negate = models.BooleanField(default=False)
order = models.IntegerField(default=0)
timeout = models.IntegerField(default=30)
objects = InheritanceManager()

View file

@ -1,7 +1,9 @@
"""passbook core policy engine"""
from logging import getLogger
from amqp.exceptions import UnexpectedFrame
from celery import group
from celery.exceptions import TimeoutError as CeleryTimeoutError
from django.core.cache import cache
from ipware import get_client_ip
@ -45,6 +47,7 @@ class PolicyEngine:
__cached = None
policies = None
__get_timeout = 0
__request = None
__user = None
@ -83,10 +86,17 @@ class PolicyEngine:
cached_policies.append(cached_policy)
else:
LOGGER.debug("Evaluating policy %s", policy.pk.hex)
signatures.append(_policy_engine_task.s(self.__user.pk, policy.pk.hex, **kwargs))
signatures.append(_policy_engine_task.signature(
args=(self.__user.pk, policy.pk.hex),
kwargs=kwargs,
time_limit=policy.timeout))
self.__get_timeout += policy.timeout
LOGGER.debug("Set total policy timeout to %r", self.__get_timeout)
# If all policies are cached, we have an empty list here.
if signatures:
self.__group = group(signatures)()
self.__get_timeout += 3
self.__get_timeout = (self.__get_timeout / len(self.policies)) * 1.5
self.__cached = cached_policies
return self
@ -98,10 +108,15 @@ class PolicyEngine:
try:
if self.__group:
# ValueError can be thrown from _policy_engine_task when user is None
result += self.__group.get()
result += self.__group.get(timeout=self.__get_timeout)
result += self.__cached
except ValueError as exc:
return False, str(exc)
# ValueError can be thrown from _policy_engine_task when user is None
return False, [str(exc)]
except UnexpectedFrame as exc:
return False, [str(exc)]
except CeleryTimeoutError as exc:
return False, [str(exc)]
for policy_action, policy_result, policy_message in result:
passing = (policy_action == Policy.ACTION_ALLOW and policy_result) or \
(policy_action == Policy.ACTION_DENY and not policy_result)

View file

@ -34,7 +34,7 @@ SECRET_KEY = CONFIG.get('secret_key')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = CONFIG.get('debug')
INTERNAL_IPS = ['127.0.0.1']
ALLOWED_HOSTS = CONFIG.get('domains', [])
ALLOWED_HOSTS = CONFIG.get('domains', []) + [CONFIG.get('primary_domain')]
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
LOGIN_URL = 'passbook_core:auth-login'
@ -45,6 +45,7 @@ AUTH_USER_MODEL = 'passbook_core.User'
CSRF_COOKIE_NAME = 'passbook_csrf'
SESSION_COOKIE_NAME = 'passbook_session'
SESSION_COOKIE_DOMAIN = CONFIG.get('primary_domain')
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
LANGUAGE_COOKIE_NAME = 'passbook_language'
@ -81,6 +82,7 @@ INSTALLED_APPS = [
'passbook.pretend.apps.PassbookPretendConfig',
'passbook.password_expiry_policy.apps.PassbookPasswordExpiryPolicyConfig',
'passbook.suspicious_policy.apps.PassbookSuspiciousPolicyConfig',
'passbook.app_gw.apps.PassbookApplicationApplicationGatewayConfig',
]
# Message Tag fix for bootstrap CSS Classes
@ -112,11 +114,12 @@ CACHES = {
}
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'passbook.app_gw.middleware.ApplicationGatewayMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',

View file

@ -195,3 +195,7 @@ form .form-row p.datetime {
.selector-remove {
background: url(../admin/img/selector-icons.svg) 0 -64px no-repeat;
}
input[data-is-monospace] {
font-family: monospace;
}

View file

@ -1,5 +1,4 @@
"""passbook core utils view"""
from django.contrib.auth.mixins import LoginRequiredMixin
from django.utils.translation import ugettext as _
from django.views.generic import TemplateView
@ -21,7 +20,7 @@ class LoadingView(TemplateView):
kwargs['target_url'] = self.get_url()
return super().get_context_data(**kwargs)
class PermissionDeniedView(LoginRequiredMixin, TemplateView):
class PermissionDeniedView(TemplateView):
"""Generic Permission denied view"""
template_name = 'login/denied.html'

View file

@ -35,6 +35,8 @@ redis: localhost/0
error_report_enabled: true
secret_key: 9$@r!d^1^jrn#fk#1#@ks#9&i$^s#1)_13%$rwjrhd=e8jfi_s
primary_domain: 'localhost'
passbook:
sign_up:
# Enables signup, created users are stored in internal Database and created in LDAP if ldap.create_users is true

View file

@ -7,3 +7,4 @@
-r passbook/captcha_factor/requirements.txt
-r passbook/admin/requirements.txt
-r passbook/api/requirements.txt
-r passbook/app_gw/requirements.txt