From 9fb1ac98eccd3a5fd5a4ffbb72c479181393fd6b Mon Sep 17 00:00:00 2001 From: Jens L Date: Sat, 3 Oct 2020 20:36:36 +0200 Subject: [PATCH] Backup/Restore (#256) * lifecycle: move s3 backup settings to s3 name * providers/oauth2: fix for alerting for missing certificatekeypair * lifecycle: add backup commands see #252 * lifecycle: install postgres-client for 11 and 12 * root: migrate to DBBACKUP_STORAGE_OPTIONS, add region setting * lifecycle: auto-clean last backups * helm: add s3 region parameter, add cronjob for backups * docs: add backup docs * root: remove backup scheduled task for now --- Dockerfile | 6 +- docker-compose.yml | 4 ++ docs/installation/kubernetes.md | 27 +++---- docs/maintenance/backups/index.md | 111 +++++++++++++++++++++++++++++ helm/templates/configmap.yaml | 9 +-- helm/templates/cronjob-backup.yaml | 42 +++++++++++ helm/values.yaml | 25 +++---- lifecycle/bootstrap.sh | 8 ++- mkdocs.yml | 2 + passbook/lib/tasks.py | 14 ---- passbook/providers/oauth2/forms.py | 7 +- passbook/root/settings.py | 32 +++++---- 12 files changed, 225 insertions(+), 62 deletions(-) create mode 100644 docs/maintenance/backups/index.md create mode 100644 helm/templates/cronjob-backup.yaml delete mode 100644 passbook/lib/tasks.py diff --git a/Dockerfile b/Dockerfile index 5803f4ae7..8e5919cd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,11 @@ COPY --from=locker /app/requirements.txt / COPY --from=locker /app/requirements-dev.txt / RUN apt-get update && \ - apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \ + apt-get install -y --no-install-recommends curl ca-certificates gnupg && \ + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ + echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends postgresql-client-12 postgresql-client-11 build-essential && \ apt-get clean && \ pip install -r /requirements.txt --no-cache-dir && \ apt-get remove --purge -y build-essential && \ diff --git a/docker-compose.yml b/docker-compose.yml index ebb851613..bcf0db10c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,8 @@ services: - traefik.port=8000 - traefik.docker.network=internal - traefik.frontend.rule=PathPrefix:/ + volumes: + - ./backups:/backups env_file: - .env worker: @@ -50,6 +52,8 @@ services: PASSBOOK_REDIS__HOST: redis PASSBOOK_POSTGRESQL__HOST: postgresql PASSBOOK_POSTGRESQL__PASSWORD: ${PG_PASS} + volumes: + - ./backups:/backups env_file: - .env static: diff --git a/docs/installation/kubernetes.md b/docs/installation/kubernetes.md index 586a3714e..7fbff3984 100644 --- a/docs/installation/kubernetes.md +++ b/docs/installation/kubernetes.md @@ -4,7 +4,7 @@ For a mid to high-load installation, Kubernetes is recommended. passbook is inst This installation automatically applies database migrations on startup. After the installation is done, you can use `pbadmin` as username and password. -``` +```yaml ################################### # Values directly affecting passbook ################################### @@ -35,8 +35,21 @@ config: # access_key: access-key # secret_key: secret-key # bucket: s3-bucket +# region: eu-central-1 # host: s3-host +ingress: + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + hosts: + - passbook.k8s.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - passbook.k8s.local + ################################### # Values controlling dependencies ################################### @@ -57,16 +70,4 @@ redis: enabled: false # https://stackoverflow.com/a/59189742 disableCommands: [] - -ingress: - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - path: / - hosts: - - passbook.k8s.local - tls: [] - # - secretName: chart-example-tls - # hosts: - # - passbook.k8s.local ``` diff --git a/docs/maintenance/backups/index.md b/docs/maintenance/backups/index.md new file mode 100644 index 000000000..ed946b101 --- /dev/null +++ b/docs/maintenance/backups/index.md @@ -0,0 +1,111 @@ +# Backup and restore + +!!! warning + + Local backups are only supported for docker-compose installs. If you want to backup a Kubernetes instance locally, use an S3-compatible server such as [minio](https://min.io/) + +### Backup + +Local backups can be created by running the following command in your passbook installation directory + +``` +docker-compose run --rm server backup +``` + +This will dump the current database into the `./backups` folder. By defaults, the last 10 Backups are kept. + +To schedule these backups, use the following snippet in a crontab + +``` +0 0 * * * bash -c "cd && docker-compose run --rm server backup" >/dev/null +``` + +!!! notice + + passbook does support automatic backups on a schedule, however this is currently not recommended, as there is no way to monitor these scheduled tasks. + +### Restore + +Run this command in your passbook installation directory + +``` +docker-compose run --rm server backup +``` + +This will prompt you to restore from your last backup. If you want to restore from a specific file, use the `-i` flag with the filename: + +``` +docker-compose run --rm server backup -i default-2020-10-03-115557.psql +``` + +After you've restored the backup, it is recommended to restart all services with `docker-compose restart`. + +### S3 Configuration + +!!! notice + + To trigger backups with S3 enabled, use the same commands as above. + +#### S3 Preparation + +passbook expects the bucket you select to already exist. The IAM User given to passbook should have the following permissions + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObjectAcl", + "s3:GetObject", + "s3:ListBucket", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Principal": { + "AWS": "arn:aws:iam::example-AWS-account-ID:user/example-user-name" + }, + "Resource": [ + "arn:aws:s3:::example-bucket-name/*", + "arn:aws:s3:::example-bucket-name" + ] + } + ] +} +``` + +#### docker-compose + +Set the following values in your `.env` file. + +``` +PASSBOOK_POSTGRESQL__S3_BACKUP__ACCESS_KEY= +PASSBOOK_POSTGRESQL__S3_BACKUP__SECRET_KEY= +PASSBOOK_POSTGRESQL__S3_BACKUP__BUCKET= +PASSBOOK_POSTGRESQL__S3_BACKUP__REGION= +``` + +If you want to backup to an S3-compatible server, like [minio](https://min.io/), use this setting: + +``` +PASSBOOK_POSTGRESQL__S3_BACKUP__HOST=http://play.min.io +``` + +#### Kubernetes + +Simply enable these options in your values.yaml file + +```yaml +# Enable Database Backups to S3 +backup: + access_key: access-key + secret_key: secret-key + bucket: s3-bucket + region: eu-central-1 + host: s3-host +``` + +Afterwards, run a `helm upgrade` to update the ConfigMap. Because passbook-scheduled backups are not recommended currently, a Kubernetes CronJob is created that runs the backup daily. diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 9de2e3ad2..414953af4 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -7,10 +7,11 @@ data: POSTGRESQL__NAME: "{{ .Values.postgresql.postgresqlDatabase }}" POSTGRESQL__USER: "{{ .Values.postgresql.postgresqlUsername }}" {{- if .Values.backup }} - POSTGRESQL__BACKUP__ACCESS_KEY: "{{ .Values.backup.access_key }}" - POSTGRESQL__BACKUP__SECRET_KEY: "{{ .Values.backup.secret_key }}" - POSTGRESQL__BACKUP__BUCKET: "{{ .Values.backup.bucket }}" - POSTGRESQL__BACKUP__HOST: "{{ .Values.backup.host }}" + POSTGRESQL__S3_BACKUP__ACCESS_KEY: "{{ .Values.backup.access_key }}" + POSTGRESQL__S3_BACKUP__SECRET_KEY: "{{ .Values.backup.secret_key }}" + POSTGRESQL__S3_BACKUP__BUCKET: "{{ .Values.backup.bucket }}" + POSTGRESQL__S3_BACKUP__REGION: "{{ .Values.backup.region }}" + POSTGRESQL__S3_BACKUP__HOST: "{{ .Values.backup.host }}" {{- end}} REDIS__HOST: "{{ .Release.Name }}-redis-master" ERROR_REPORTING__ENABLED: "{{ .Values.config.error_reporting.enabled }}" diff --git a/helm/templates/cronjob-backup.yaml b/helm/templates/cronjob-backup.yaml new file mode 100644 index 000000000..5a06a7d07 --- /dev/null +++ b/helm/templates/cronjob-backup.yaml @@ -0,0 +1,42 @@ +{{- if .Values.backup }} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ include "passbook.fullname" . }}-backup + labels: + app.kubernetes.io/name: {{ include "passbook.name" . }} + helm.sh/chart: {{ include "passbook.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + schedule: "0 0 * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.name }}:{{ .Values.image.tag }}" + args: [server] + envFrom: + - configMapRef: + name: {{ include "passbook.fullname" . }}-config + prefix: PASSBOOK_ + env: + - name: PASSBOOK_SECRET_KEY + valueFrom: + secretKeyRef: + name: "{{ include "passbook.fullname" . }}-secret-key" + key: "secret_key" + - name: PASSBOOK_REDIS__PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-redis" + key: "redis-password" + - name: PASSBOOK_POSTGRESQL__PASSWORD + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-postgresql" + key: "postgresql-password" +{{- end}} diff --git a/helm/values.yaml b/helm/values.yaml index c85cdcdbf..ed76ad1be 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -28,8 +28,21 @@ config: # access_key: access-key # secret_key: secret-key # bucket: s3-bucket +# region: eu-central-1 # host: s3-host +ingress: + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + hosts: + - passbook.k8s.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - passbook.k8s.local + ################################### # Values controlling dependencies ################################### @@ -50,15 +63,3 @@ redis: enabled: false # https://stackoverflow.com/a/59189742 disableCommands: [] - -ingress: - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - path: / - hosts: - - passbook.k8s.local - tls: [] - # - secretName: chart-example-tls - # hosts: - # - passbook.k8s.local diff --git a/lifecycle/bootstrap.sh b/lifecycle/bootstrap.sh index 21a3b8f15..46cc932fa 100755 --- a/lifecycle/bootstrap.sh +++ b/lifecycle/bootstrap.sh @@ -1,6 +1,6 @@ #!/bin/bash -e python -m lifecycle.wait_for_db -printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@" +printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@" > /dev/stderr if [[ "$1" == "server" ]]; then gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application elif [[ "$1" == "worker" ]]; then @@ -9,6 +9,12 @@ elif [[ "$1" == "migrate" ]]; then # Run system migrations first, run normal migrations after python -m lifecycle.migrate python -m manage migrate +elif [[ "$1" == "backup" ]]; then + python -m manage dbbackup --clean +elif [[ "$1" == "restore" ]]; then + python -m manage dbrestore ${@:2} +elif [[ "$1" == "bash" ]]; then + /bin/bash else python -m manage "$@" fi diff --git a/mkdocs.yml b/mkdocs.yml index 036eba2b3..8915df5b9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -57,6 +57,8 @@ nav: - Ubuntu Landscape: integrations/services/ubuntu-landscape/index.md - Sonarr: integrations/services/sonarr/index.md - Tautulli: integrations/services/tautulli/index.md + - Maintenance: + - Backups: maintenance/backups/index.md - Upgrading: - to 0.9: upgrading/to-0.9.md - to 0.10: upgrading/to-0.10.md diff --git a/passbook/lib/tasks.py b/passbook/lib/tasks.py deleted file mode 100644 index 8c451fc65..000000000 --- a/passbook/lib/tasks.py +++ /dev/null @@ -1,14 +0,0 @@ -"""passbook misc tasks""" -from django.core import management -from structlog import get_logger - -from passbook.root.celery import CELERY_APP - -LOGGER = get_logger() - - -@CELERY_APP.task() -def backup_database(): # pragma: no cover - """Backup database""" - management.call_command("dbbackup") - LOGGER.info("Successfully backed up database.") diff --git a/passbook/providers/oauth2/forms.py b/passbook/providers/oauth2/forms.py index 1bd5e257f..c699659e2 100644 --- a/passbook/providers/oauth2/forms.py +++ b/passbook/providers/oauth2/forms.py @@ -12,7 +12,7 @@ from passbook.providers.oauth2.generators import ( generate_client_id, generate_client_secret, ) -from passbook.providers.oauth2.models import OAuth2Provider, ScopeMapping +from passbook.providers.oauth2.models import JWTAlgorithms, OAuth2Provider, ScopeMapping class OAuth2ProviderForm(forms.ModelForm): @@ -32,7 +32,10 @@ class OAuth2ProviderForm(forms.ModelForm): def clean_jwt_alg(self): """Ensure that when RS256 is selected, a certificate-key-pair is selected""" - if "rsa_key" not in self.cleaned_data: + if ( + self.data["rsa_key"] == "" + and self.cleaned_data["jwt_alg"] == JWTAlgorithms.RS256 + ): raise ValidationError( _("RS256 requires a Certificate-Key-Pair to be selected.") ) diff --git a/passbook/root/settings.py b/passbook/root/settings.py index d0d0e7bc6..d5a2bf410 100644 --- a/passbook/root/settings.py +++ b/passbook/root/settings.py @@ -272,7 +272,7 @@ CELERY_BEAT_SCHEDULE = { "options": {"queue": "passbook_scheduled"}, } } -CELERY_CREATE_MISSING_QUEUES = True +CELERY_TASK_CREATE_MISSING_QUEUES = True CELERY_TASK_DEFAULT_QUEUE = "passbook" CELERY_BROKER_URL = ( f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}" @@ -284,24 +284,25 @@ CELERY_RESULT_BACKEND = ( ) # Database backup -if CONFIG.y("postgresql.backup"): +DBBACKUP_STORAGE = "django.core.files.storage.FileSystemStorage" +DBBACKUP_STORAGE_OPTIONS = {"location": "./backups" if DEBUG else "/backups"} +DBBACKUP_CONNECTOR_MAPPING = { + "django_prometheus.db.backends.postgresql": "dbbackup.db.postgresql.PgDumpConnector" +} +if CONFIG.y("postgresql.s3_backup"): DBBACKUP_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" - DBBACKUP_CONNECTOR_MAPPING = { - "django_prometheus.db.backends.postgresql": "dbbackup.db.postgresql.PgDumpConnector" + DBBACKUP_STORAGE_OPTIONS = { + "access_key": CONFIG.y("postgresql.s3_backup.access_key"), + "secret_key": CONFIG.y("postgresql.s3_backup.secret_key"), + "bucket_name": CONFIG.y("postgresql.s3_backup.bucket"), + "region_name": CONFIG.y("postgresql.s3_backup.region", "eu-central-1"), + "default_acl": "private", + "endpoint_url": CONFIG.y("postgresql.s3_backup.host"), } - AWS_ACCESS_KEY_ID = CONFIG.y("postgresql.backup.access_key") - AWS_SECRET_ACCESS_KEY = CONFIG.y("postgresql.backup.secret_key") - AWS_STORAGE_BUCKET_NAME = CONFIG.y("postgresql.backup.bucket") - AWS_S3_ENDPOINT_URL = CONFIG.y("postgresql.backup.host") - AWS_DEFAULT_ACL = None j_print( - "Database backup to S3 is configured.", host=CONFIG.y("postgresql.backup.host") + "Database backup to S3 is configured.", + host=CONFIG.y("postgresql.s3_backup.host"), ) - # Add automatic task to backup - CELERY_BEAT_SCHEDULE["db_backup"] = { - "task": "passbook.lib.tasks.backup_database", - "schedule": crontab(minute=0, hour=0), # Run every day, midnight - } # Sentry integration _ERROR_REPORTING = CONFIG.y_bool("error_reporting.enabled", False) @@ -400,6 +401,7 @@ _LOGGING_HANDLER_MAP = { "urllib3": "WARNING", "websockets": "WARNING", "daphne": "WARNING", + "dbbackup": "ERROR", } for handler_name, level in _LOGGING_HANDLER_MAP.items(): # pyright: reportGeneralTypeIssues=false