From f0fadf8bba7d15da01f5b4949a1c8eaaee856f77 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 27 Jan 2021 15:49:02 +0100 Subject: [PATCH 01/22] modify Docker --- INSTALLDEV.md | 2 +- scripts/containers/Dockerfile | 4 +- scripts/containers/orchestra-admin | 247 +++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 scripts/containers/orchestra-admin diff --git a/INSTALLDEV.md b/INSTALLDEV.md index 93f59f00..caa8a5b8 100644 --- a/INSTALLDEV.md +++ b/INSTALLDEV.md @@ -11,7 +11,7 @@ If you are planing to do some development you may want to consider doing it unde 2. Build a new image, create and start a container ```bash - curl -L http://git.io/orchestra-Dockerfile > /tmp/Dockerfile + curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/Dockerfile > /tmp/Dockerfile docker build -t orchestra /tmp/ docker create --name orchestra -i -t -u orchestra -w /home/orchestra orchestra bash docker start orchestra diff --git a/scripts/containers/Dockerfile b/scripts/containers/Dockerfile index b96691c9..85535700 100644 --- a/scripts/containers/Dockerfile +++ b/scripts/containers/Dockerfile @@ -1,8 +1,8 @@ -FROM debian:latest +FROM debian:9-slim RUN apt-get -y update && apt-get install -y curl sudo -RUN export TERM=xterm; curl -L http://git.io/orchestra-admin | bash -s install_requirements +RUN export TERM=urxvt; curl -L http://git.io/orchestra-admin | bash -s install_requirements RUN apt-get clean diff --git a/scripts/containers/orchestra-admin b/scripts/containers/orchestra-admin new file mode 100644 index 00000000..48568438 --- /dev/null +++ b/scripts/containers/orchestra-admin @@ -0,0 +1,247 @@ +#!/bin/bash + +set -u +set -e + +bold=$(tput -T ${TERM:-xterm} bold) +normal=$(tput -T ${TERM:-xterm} sgr0) + + +PYTHON_BIN='python3' + +function help () { + if [[ $# -gt 1 ]]; then + CMD="print_${2}_help" + $CMD + else + print_help + fi +} + + +function print_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchestra-admin${normal} - Orchetsra administration script + + ${bold}OPTIONS${normal} + ${bold}install_requirements${normal} + Installs Orchestra requirements using apt-get and pip + + ${bold}startproject${normal} + Creates a new Django-orchestra instance + + ${bold}help${normal} + Displays this help text or related help page as argument + for example: + ${bold}orchestra-admin help startproject${normal} + + EOF +} + + +show () { + echo " ${bold}\$ ${@}${normal}" +} +export -f show + + +run () { + show "${@}" + "${@}" +} +export -f run + + +check_root () { + [ $(whoami) != 'root' ] && { echo -e "\nErr. This should be run as root\n" >&2; exit 1; } +} +export -f check_root + + +get_orchestra_dir () { + if ! $(echo "import orchestra" | $PYTHON_BIN 2> /dev/null); then + echo -e "\norchestra not installed.\n" >&2 + exit 1 + fi + PATH=$(echo "import orchestra, os; print(os.path.dirname(os.path.realpath(orchestra.__file__)))" | $PYTHON_BIN) + echo $PATH +} +export -f get_orchestra_dir + + +function print_install_requirements_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchetsra-admin install_requirements${normal} - Installs all Orchestra requirements using apt-get and pip + + ${bold}OPTIONS${normal} + ${bold}-t, --testing${normal} + Install Orchestra normal requirements plus those needed for running functional tests + + ${bold}-h, --help${normal} + Displays this help text + + EOF +} + + +function install_requirements () { + opts=$(getopt -o h,t -l help,testing -- "$@") || exit 1 + set -- $opts + testing=false + + while [ $# -gt 0 ]; do + case $1 in + -h|--help) print_deploy_help; exit 0 ;; + -t|--testing) testing=true; shift ;; + (--) shift; break;; + (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift + done + unset OPTIND + unset opt + + check_root || true + ORCHESTRA_PATH=$(get_orchestra_dir) || true + + # Make sure locales are in place before installing postgres + if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then + run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen + run locale-gen + update-locale LANG=en_US.UTF-8 + fi + + # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev + APT="bind9utils \ + ca-certificates \ + gettext \ + libcrack2-dev \ + libxml2-dev \ + libxslt1-dev \ + python3 \ + python3-pip \ + python3-dev \ + ssh-client \ + wget \ + xvfb \ + zlib1g-dev" + if $testing; then + APT="${APT} \ + git \ + iceweasel \ + dnsutils" + fi + + run apt-get update + run apt-get install -y $APT + + # Install ca certificates before executing pip install + if [[ ! -e /usr/local/share/ca-certificates/cacert.org ]]; then + mkdir -p /usr/local/share/ca-certificates/cacert.org + wget -P /usr/local/share/ca-certificates/cacert.org \ + http://www.cacert.org/certs/root.crt \ + http://www.cacert.org/certs/class3.crt + update-ca-certificates + fi + + # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies + PIP="$(wget http://git.io/orchestra-requirements.txt -O - | tr '\n' ' ') \ + cracklib \ + lxml==3.3.5" + if $testing; then + PIP="${PIP} \ + selenium \ + xvfbwrapper \ + freezegun \ + coverage \ + flake8 \ + django-debug-toolbar==1.3.0 \ + django-nose==1.4.4 \ + sqlparse \ + pyinotify \ + PyMySQL" + fi + + run pip3 install $PIP + + # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support) + wkhtmltox_version=$(dpkg --list | grep wkhtmltox | awk {'print $3'}) + minor=$(echo -e "$wkhtmltox_version\n0.12.2.1" | sort -V | head -n 1) + if [[ ! $wkhtmltox_version ]] || [[ $wkhtmltox_version != 0.12.2.1 && $minor == ${wkhtmltox_version} ]]; then + wkhtmltox=$(mktemp) + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.stretch_amd64.deb -O ${wkhtmltox} + #wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb -O ${wkhtmltox} + dpkg -i ${wkhtmltox} || { echo "Installing missing dependencies for wkhtmltox..." && apt-get -f -y install; } + fi +} +export -f install_requirements + + +print_startproject_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}orchestra-admin startproject${normal} - Create a new Django-Orchestra instance + + ${bold}SYNOPSIS${normal} + Options: [ -h ] + + ${bold}OPTIONS${normal} + ${bold}-h, --help${normal} + This help message + + ${bold}EXAMPLES${normal} + orchestra-admin startproject controlpanel + + EOF +} + + +function startproject () { + local PROJECT_NAME="$2"; shift + + opts=$(getopt -o h -l help -- "$@") || exit 1 + set -- $opts + + set -- $opts + while [ $# -gt 0 ]; do + case $1 in + -h|--help) print_startproject_help; exit 0 ;; + (--) shift; break;; + (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift + done + + unset OPTIND + unset opt + + [ $(whoami) == 'root' ] && { echo -e "\nYou don't want to run this as root\n" >&2; exit 1; } + ORCHESTRA_PATH=$(get_orchestra_dir) || { echo "Error getting orchestra dir"; exit 1; } + if [[ ! -e $PROJECT_NAME/manage.py ]]; then + run django-admin.py startproject $PROJECT_NAME --template="${ORCHESTRA_PATH}/conf/project_template" + # This is a workaround for this issue https://github.com/pypa/pip/issues/317 + run chmod +x $PROJECT_NAME/manage.py + # End of workaround ### + else + echo "Not cloning: $PROJECT_NAME already exists." + fi + # Install bash autocompletition for django commands + if [[ ! $(grep 'source $HOME/.django_bash_completion.sh' ~/.bashrc &> /dev/null) ]]; then + # run wget https://raw.github.com/django/django/master/extras/django_bash_completion \ + # --no-check-certificate -O ~/.django_bash_completion.sh + cp ${ORCHESTRA_PATH}/bin/django_bash_completion.sh ~/.django_bash_completion.sh + echo 'source $HOME/.django_bash_completion.sh' >> ~/.bashrc + fi +} +export -f startproject + + +[ $# -lt 1 ] && { print_help; exit 1; } +$1 "${@}" From e2ef8823f8364c805bcf74eee2cad6f8dc27316f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 27 Jan 2021 15:51:16 +0100 Subject: [PATCH 02/22] modify source of orchestra-admin --- scripts/containers/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/containers/Dockerfile b/scripts/containers/Dockerfile index 85535700..63cb9d3d 100644 --- a/scripts/containers/Dockerfile +++ b/scripts/containers/Dockerfile @@ -2,7 +2,7 @@ FROM debian:9-slim RUN apt-get -y update && apt-get install -y curl sudo -RUN export TERM=urxvt; curl -L http://git.io/orchestra-admin | bash -s install_requirements +RUN export TERM=xterm; curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-admin | bash -s install_requirements RUN apt-get clean From 44ebd42942c5af2bf6b197413718efa5164a3ca6 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 12:20:45 +0100 Subject: [PATCH 03/22] adapt docker env for new version of debian --- INSTALLDEV.md | 4 +- install_manually.md | 132 +++++++++++++++ orchestra/bin/orchestra-admin | 2 +- scripts/containers/Dockerfile | 134 +++++++++++++++- scripts/containers/orchestra-admin | 247 ----------------------------- 5 files changed, 268 insertions(+), 251 deletions(-) create mode 100644 install_manually.md delete mode 100644 scripts/containers/orchestra-admin diff --git a/INSTALLDEV.md b/INSTALLDEV.md index caa8a5b8..96420d12 100644 --- a/INSTALLDEV.md +++ b/INSTALLDEV.md @@ -21,7 +21,7 @@ If you are planing to do some development you may want to consider doing it unde 3. Deploy django-orchestra development environment, inside the container ```bash - bash <( curl -L http://git.io/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-deploy ) --dev ``` 3. Nginx should be serving on port 80, but Django's development server can be used as well: @@ -34,5 +34,5 @@ If you are planing to do some development you may want to consider doing it unde 5. To upgrade to current master just re-run the deploy script ```bash git pull origin master - bash <( curl -L http://git.io/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-deploy ) --dev ``` diff --git a/install_manually.md b/install_manually.md new file mode 100644 index 00000000..a2747041 --- /dev/null +++ b/install_manually.md @@ -0,0 +1,132 @@ +# System requirements: +The most important requirement is use python3.6 +we need install this packages: +``` +bind9utils +ca-certificates +gettext +libcrack2-dev +libxml2-dev +libxslt1-dev +python3 +python3-pip +python3-dev +ssh-client +wget +xvfb +zlib1g-dev +git +iceweasel +dnsutils +``` +We need install too a *wkhtmltopdf* package +You can use one of your OS or get it from original. +This it is in https://wkhtmltopdf.org/downloads.html + +# pip installations +We need install this packages: +``` +Django==1.10.5 +django-fluent-dashboard==0.6.1 +django-admin-tools==0.8.0 +django-extensions==1.7.4 +django-celery==3.1.17 +celery==3.1.23 +kombu==3.0.35 +billiard==3.3.0.23 +Markdown==2.4 +djangorestframework==3.4.7 +ecdsa==0.11 +Pygments==1.6 +django-filter==0.15.2 +jsonfield==0.9.22 +python-dateutil==2.2 +https://github.com/glic3rinu/passlib/archive/master.zip +django-iban==0.3.0 +requests +phonenumbers +django-countries +django-localflavor +amqp +anyjson +pytz +cracklib +lxml==3.3.5 +selenium +xvfbwrapper +freezegun +coverage +flake8 +django-debug-toolbar==1.3.0 +django-nose==1.4.4 +sqlparse +pyinotify +PyMySQL +``` + +If you want to use Orchestra you need to install from pip like this: +``` +pip3 install http://git.io/django-orchestra-dev +``` + +But if you want develop orquestra you need to do this: +``` +git clone https://github.com/ribaguifi/django-orchestra +pip install -e django-orchestra +``` + +# Database +For default use sqlite3 if you want to use postgresql you need install this packages: + +``` +psycopg2 postgresql +``` + +You can use it for debian or ubuntu: + +``` +sudo apt-get install python3-psycopg2 postgresql-contrib +``` + +Remember create a database for your project and give permitions for the correct user like this: + +``` +psql -U postgres +psql (12.4) +Digite «help» para obtener ayuda. + +postgres=# CREATE database orchesta; +postgres=# CREATE USER orchesta WITH PASSWORD 'orquesta'; +postgres=# GRANT ALL PRIVILEGES ON DATABASE orchesta TO orchesta; +``` + +# Create new project +You can use orchestra-admin for create your new project +``` +orchestra-admin startproject # e.g. panel +cd +``` + +Next we need change the settings.py for configure the correct database + +In settings.py we need change the DATABASE section like this: + +``` +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'orchestra' + 'USER': 'orchestra', + 'PASSWORD': 'orchestra', + 'HOST': 'localhost', + 'PORT': '5432', + 'CONN_MAX_AGE': 60*10 + } +} +``` + +For end you need to do the migrations: + +``` +python3 manage.py migrate +``` diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index 22a26e72..82dba1b9 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -174,7 +174,7 @@ function install_requirements () { minor=$(echo -e "$wkhtmltox_version\n0.12.2.1" | sort -V | head -n 1) if [[ ! $wkhtmltox_version ]] || [[ $wkhtmltox_version != 0.12.2.1 && $minor == ${wkhtmltox_version} ]]; then wkhtmltox=$(mktemp) - wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb -O ${wkhtmltox} + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_amd64.deb -O ${wkhtmltox} dpkg -i ${wkhtmltox} || { echo "Installing missing dependencies for wkhtmltox..." && apt-get -f -y install; } fi } diff --git a/scripts/containers/Dockerfile b/scripts/containers/Dockerfile index 63cb9d3d..b137a261 100644 --- a/scripts/containers/Dockerfile +++ b/scripts/containers/Dockerfile @@ -1,4 +1,134 @@ -FROM debian:9-slim +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "update.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + +FROM buildpack-deps:buster + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# extra dependencies (over what buildpack-deps already includes) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + && rm -rf /var/lib/apt/lists/* + +ENV GPG_KEY 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D +ENV PYTHON_VERSION 3.6.12 + +RUN set -ex \ + \ + && wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ + && wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$GPG_KEY" \ + && gpg --batch --verify python.tar.xz.asc python.tar.xz \ + && { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \ + && rm -rf "$GNUPGHOME" python.tar.xz.asc \ + && mkdir -p /usr/src/python \ + && tar -xJC /usr/src/python --strip-components=1 -f python.tar.xz \ + && rm python.tar.xz \ + \ + && cd /usr/src/python \ + && gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" \ + && ./configure \ + --build="$gnuArch" \ + --enable-loadable-sqlite-extensions \ + --enable-optimizations \ + --enable-option-checking=fatal \ + --enable-shared \ + --with-system-expat \ + --with-system-ffi \ + --without-ensurepip \ + && make -j "$(nproc)" \ +# setting PROFILE_TASK makes "--enable-optimizations" reasonable: https://bugs.python.org/issue36044 / https://github.com/docker-library/python/issues/160#issuecomment-509426916 + PROFILE_TASK='-m test.regrtest --pgo \ + test_array \ + test_base64 \ + test_binascii \ + test_binhex \ + test_binop \ + test_bytes \ + test_c_locale_coercion \ + test_class \ + test_cmath \ + test_codecs \ + test_compile \ + test_complex \ + test_csv \ + test_decimal \ + test_dict \ + test_float \ + test_fstring \ + test_hashlib \ + test_io \ + test_iter \ + test_json \ + test_long \ + test_math \ + test_memoryview \ + test_pickle \ + test_re \ + test_set \ + test_slice \ + test_struct \ + test_threading \ + test_time \ + test_traceback \ + test_unicode \ + ' \ + && make install \ + && rm -rf /usr/src/python \ + \ + && find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \ + -o \( -type f -a -name 'wininst-*.exe' \) \ + \) -exec rm -rf '{}' + \ + \ + && ldconfig \ + \ + && python3 --version + +# make some useful symlinks that are expected to exist +RUN cd /usr/local/bin \ + && ln -s idle3 idle \ + && ln -s pydoc3 pydoc \ + && ln -s python3 python \ + && ln -s python3-config python-config + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 21.0 +# https://github.com/pypa/get-pip +ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/8cc88aca7d9775fce279e8b84ef163cf1d3e8a2e/get-pip.py +ENV PYTHON_GET_PIP_SHA256 ffb67da2e976f48dd29714fc64812d1ac419eb7d48079737166dd95640d1debd + +RUN set -ex; \ + \ + wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ + echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \ + \ + python get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + "pip==$PYTHON_PIP_VERSION" \ + ; \ + pip --version; \ + \ + find /usr/local -depth \ + \( \ + \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \ + -o \ + \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ + \) -exec rm -rf '{}' +; \ + rm -f get-pip.py RUN apt-get -y update && apt-get install -y curl sudo @@ -10,3 +140,5 @@ RUN useradd orchestra --shell /bin/bash && \ { echo "orchestra:orchestra" | chpasswd; } && \ mkhomedir_helper orchestra && \ adduser orchestra sudo + +CMD ["python3"] diff --git a/scripts/containers/orchestra-admin b/scripts/containers/orchestra-admin deleted file mode 100644 index 48568438..00000000 --- a/scripts/containers/orchestra-admin +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/bash - -set -u -set -e - -bold=$(tput -T ${TERM:-xterm} bold) -normal=$(tput -T ${TERM:-xterm} sgr0) - - -PYTHON_BIN='python3' - -function help () { - if [[ $# -gt 1 ]]; then - CMD="print_${2}_help" - $CMD - else - print_help - fi -} - - -function print_help () { - cat <<- EOF - - ${bold}NAME${normal} - ${bold}orchestra-admin${normal} - Orchetsra administration script - - ${bold}OPTIONS${normal} - ${bold}install_requirements${normal} - Installs Orchestra requirements using apt-get and pip - - ${bold}startproject${normal} - Creates a new Django-orchestra instance - - ${bold}help${normal} - Displays this help text or related help page as argument - for example: - ${bold}orchestra-admin help startproject${normal} - - EOF -} - - -show () { - echo " ${bold}\$ ${@}${normal}" -} -export -f show - - -run () { - show "${@}" - "${@}" -} -export -f run - - -check_root () { - [ $(whoami) != 'root' ] && { echo -e "\nErr. This should be run as root\n" >&2; exit 1; } -} -export -f check_root - - -get_orchestra_dir () { - if ! $(echo "import orchestra" | $PYTHON_BIN 2> /dev/null); then - echo -e "\norchestra not installed.\n" >&2 - exit 1 - fi - PATH=$(echo "import orchestra, os; print(os.path.dirname(os.path.realpath(orchestra.__file__)))" | $PYTHON_BIN) - echo $PATH -} -export -f get_orchestra_dir - - -function print_install_requirements_help () { - cat <<- EOF - - ${bold}NAME${normal} - ${bold}orchetsra-admin install_requirements${normal} - Installs all Orchestra requirements using apt-get and pip - - ${bold}OPTIONS${normal} - ${bold}-t, --testing${normal} - Install Orchestra normal requirements plus those needed for running functional tests - - ${bold}-h, --help${normal} - Displays this help text - - EOF -} - - -function install_requirements () { - opts=$(getopt -o h,t -l help,testing -- "$@") || exit 1 - set -- $opts - testing=false - - while [ $# -gt 0 ]; do - case $1 in - -h|--help) print_deploy_help; exit 0 ;; - -t|--testing) testing=true; shift ;; - (--) shift; break;; - (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; - (*) break;; - esac - shift - done - unset OPTIND - unset opt - - check_root || true - ORCHESTRA_PATH=$(get_orchestra_dir) || true - - # Make sure locales are in place before installing postgres - if [[ $({ perl --help > /dev/null; } 2>&1|grep 'locale failed') ]]; then - run sed -i "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen - run locale-gen - update-locale LANG=en_US.UTF-8 - fi - - # lxml: libxml2-dev, libxslt1-dev, zlib1g-dev - APT="bind9utils \ - ca-certificates \ - gettext \ - libcrack2-dev \ - libxml2-dev \ - libxslt1-dev \ - python3 \ - python3-pip \ - python3-dev \ - ssh-client \ - wget \ - xvfb \ - zlib1g-dev" - if $testing; then - APT="${APT} \ - git \ - iceweasel \ - dnsutils" - fi - - run apt-get update - run apt-get install -y $APT - - # Install ca certificates before executing pip install - if [[ ! -e /usr/local/share/ca-certificates/cacert.org ]]; then - mkdir -p /usr/local/share/ca-certificates/cacert.org - wget -P /usr/local/share/ca-certificates/cacert.org \ - http://www.cacert.org/certs/root.crt \ - http://www.cacert.org/certs/class3.crt - update-ca-certificates - fi - - # cracklib and lxml are excluded on the requirements.txt because they need unconvinient system dependencies - PIP="$(wget http://git.io/orchestra-requirements.txt -O - | tr '\n' ' ') \ - cracklib \ - lxml==3.3.5" - if $testing; then - PIP="${PIP} \ - selenium \ - xvfbwrapper \ - freezegun \ - coverage \ - flake8 \ - django-debug-toolbar==1.3.0 \ - django-nose==1.4.4 \ - sqlparse \ - pyinotify \ - PyMySQL" - fi - - run pip3 install $PIP - - # Install a more recent version of wkhtmltopdf (0.12.2) (PDF page number support) - wkhtmltox_version=$(dpkg --list | grep wkhtmltox | awk {'print $3'}) - minor=$(echo -e "$wkhtmltox_version\n0.12.2.1" | sort -V | head -n 1) - if [[ ! $wkhtmltox_version ]] || [[ $wkhtmltox_version != 0.12.2.1 && $minor == ${wkhtmltox_version} ]]; then - wkhtmltox=$(mktemp) - wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.stretch_amd64.deb -O ${wkhtmltox} - #wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/wkhtmltox-0.12.2.1_linux-jessie-amd64.deb -O ${wkhtmltox} - dpkg -i ${wkhtmltox} || { echo "Installing missing dependencies for wkhtmltox..." && apt-get -f -y install; } - fi -} -export -f install_requirements - - -print_startproject_help () { - cat <<- EOF - - ${bold}NAME${normal} - ${bold}orchestra-admin startproject${normal} - Create a new Django-Orchestra instance - - ${bold}SYNOPSIS${normal} - Options: [ -h ] - - ${bold}OPTIONS${normal} - ${bold}-h, --help${normal} - This help message - - ${bold}EXAMPLES${normal} - orchestra-admin startproject controlpanel - - EOF -} - - -function startproject () { - local PROJECT_NAME="$2"; shift - - opts=$(getopt -o h -l help -- "$@") || exit 1 - set -- $opts - - set -- $opts - while [ $# -gt 0 ]; do - case $1 in - -h|--help) print_startproject_help; exit 0 ;; - (--) shift; break;; - (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; - (*) break;; - esac - shift - done - - unset OPTIND - unset opt - - [ $(whoami) == 'root' ] && { echo -e "\nYou don't want to run this as root\n" >&2; exit 1; } - ORCHESTRA_PATH=$(get_orchestra_dir) || { echo "Error getting orchestra dir"; exit 1; } - if [[ ! -e $PROJECT_NAME/manage.py ]]; then - run django-admin.py startproject $PROJECT_NAME --template="${ORCHESTRA_PATH}/conf/project_template" - # This is a workaround for this issue https://github.com/pypa/pip/issues/317 - run chmod +x $PROJECT_NAME/manage.py - # End of workaround ### - else - echo "Not cloning: $PROJECT_NAME already exists." - fi - # Install bash autocompletition for django commands - if [[ ! $(grep 'source $HOME/.django_bash_completion.sh' ~/.bashrc &> /dev/null) ]]; then - # run wget https://raw.github.com/django/django/master/extras/django_bash_completion \ - # --no-check-certificate -O ~/.django_bash_completion.sh - cp ${ORCHESTRA_PATH}/bin/django_bash_completion.sh ~/.django_bash_completion.sh - echo 'source $HOME/.django_bash_completion.sh' >> ~/.bashrc - fi -} -export -f startproject - - -[ $# -lt 1 ] && { print_help; exit 1; } -$1 "${@}" From 30bd1ad81673dde74b924bd0db9a87f0251ca6f3 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 12:25:43 +0100 Subject: [PATCH 04/22] fixed path --- scripts/containers/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/containers/Dockerfile b/scripts/containers/Dockerfile index b137a261..35c92639 100644 --- a/scripts/containers/Dockerfile +++ b/scripts/containers/Dockerfile @@ -132,7 +132,7 @@ RUN set -ex; \ RUN apt-get -y update && apt-get install -y curl sudo -RUN export TERM=xterm; curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-admin | bash -s install_requirements +RUN export TERM=xterm; curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/orchestra/bin/orchestra-admin | bash -s install_requirements RUN apt-get clean From 0b937bfb4f8e4be35ccd0751ff98fd9cc0b0192a Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 12:34:53 +0100 Subject: [PATCH 05/22] up orchestra-deploy --- scripts/containers/orchestra-deploy | 308 ++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 scripts/containers/orchestra-deploy diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy new file mode 100644 index 00000000..a936f5dd --- /dev/null +++ b/scripts/containers/orchestra-deploy @@ -0,0 +1,308 @@ +#!/bin/bash + +set -ue + +# https://git.io/deploy-orchestra +# bash <( curl https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/scripts/containers/deploy.sh ) [--noinput username] + + +bold=$(tput -T ${TERM:-xterm} bold) +normal=$(tput -T ${TERM:-xterm} sgr0) + + +function test_orchestra () { + user=$1 + ip_addr=$2 + # Test if serving requests + echo + echo ${bold} + echo "> Checking if Orchestra is serving on https://${ip_addr}/admin/ ..." + if [[ $noinput ]]; then + echo " - username: $user" + echo " - password: orchestra${normal}" + fi + echo + if [[ $(curl -s -L -k -c /tmp/cookies.txt -b /tmp/cookies.txt https://$ip_addr/admin/ | grep 'Panel Hosting Management') ]]; then + token=$(grep csrftoken /tmp/cookies.txt | awk {'print $7'}) + if [[ $(curl -s -L -k -c /tmp/cookies.txt -b /tmp/cookies.txt \ + -d "username=$user&password=orchestra&csrfmiddlewaretoken=$token" \ + -e https://$ip_addr/admin/ \ + https://$ip_addr/admin/login/?next=/admin/ | grep 'Panel Hosting Management') ]]; then + echo "${bold} ** Orchestra appears to be working!${normal}" + else + echo "${bold} ** Err. Couldn't login :(${normal}" >&2 + fi + else + echo "${bold} ** Err. Orchestra is not responding responding on https://${ip_addr}/admin/${normal}" >&2 + fi + echo +} + + +function install_orchestra () { + dev=$1 + home=$2 + repo=$3 + + if [[ $dev ]]; then + # Install from source + python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][0]);") + if [[ -d $python_path/orchestra ]]; then + run sudo rm -fr $python_path/orchestra + fi + #orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null) + orch_version='' + if [[ ! $orch_version ]]; then + # First Orchestra installation + run sudo mkdir -p /usr/share/man/man1 + run sudo mkdir -p /usr/share/man/man7 + run sudo apt-get update + run sudo apt-get -y install git python3-pip + surun "git clone $repo $home/django-orchestra" || { + # Finishing partial installation + surun "export GIT_DIR=$home/django-orchestra/.git; git pull" + } + echo $home/django-orchestra/ | sudo tee "$python_path/orchestra.pth" + else + echo "You may want to execute 'git pull origin master'?" + fi + if [[ -L /usr/local/bin/orchestra-admin || -f /usr/local/bin/orchestra-admin ]]; then + run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} + fi + run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/ + run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-beat /usr/local/bin/ + run sudo orchestra-admin install_requirements --testing + else + # Install from pip + run sudo orchestra-admin install_requirements + #run sudo pip3 install http://git.io/django-orchestra-dev + run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git + fi +} + + +function setup_database () { + dev=$1 + noinput=$2 + run sudo apt-get install -y postgresql python3-psycopg2 + # Setup Database + if [[ $dev ]]; then + # Speeding up tests, don't do this in production! + . /usr/share/postgresql-common/init.d-functions + pg_version=$(psql --version | head -n1 | sed -r "s/^.*\s([0-9]+\.[0-9]+).*/\1/") + sudo sed -i \ + -e "s/^#fsync =\s*.*/fsync = off/" \ + -e "s/^#full_page_writes =\s*.*/full_page_writes = off/" \ + /etc/postgresql/${pg_version}/main/postgresql.conf + fi + run sudo service postgresql start + if [[ $noinput ]]; then + db_password=$(ps aux | sha256sum | base64 | head -c 10) + run sudo python3 -W ignore manage.py setuppostgres --noinput --db_password $db_password + else + run sudo python3 -W ignore manage.py setuppostgres + fi + if [[ $dev ]]; then + # Create database permissions are needed for running tests + sudo su postgres -c 'psql -c "ALTER USER orchestra CREATEDB;"' + fi + surun "python3 -W ignore manage.py migrate $noinput" +} + + +function create_orchestra_superuser () { + user=$1 + email=$2 + password=$3 + cat <<- EOF | surun "python3 -W ignore manage.py shell" + from orchestra.contrib.accounts.models import Account + if not Account.objects.filter(username="$user").exists(): + print('Creating orchestra superuser') + Account.objects.create_superuser("$user", "$email", "$password") + + EOF +} + + +print_help () { + cat <<- EOF + + ${bold}NAME${normal} + ${bold}deploy.sh${normal} - Deploy a django-orchestra project + + ${bold}SYNOPSIS${normal} + ${bold}deploy.sh${normal} [--noinput=USERNAME] [--dev] [--repo=GITREPO] [--projectname=NAME] + + ${bold}OPTIONS${normal} + ${bold}-n, --noinput=USERNAME${normal} + Execute the script without any user input, an existing system USERNAME is required. + requires the script to be executed as root user + + ${bold}-d, --dev${normal} + Perform a deployment suitable for development: + 1. debug mode + 2. dependencies for running tests + 3. access to source code + + ${bold}-r, --repo=GITREPO${normal} + Chose which repo use for development deployment + this option requires --dev option to be selected + https://github.com/glic3rinu/django-orchestra.git is used by default + + ${bold}-p, --projectname=NAME${normal} + Specify a project name, this will be asked on interactive mode + and name 'panel' will be used otherwise. + + ${bold}-h, --help${normal} + Display this message + + ${bold}EXAMPLES${normal} + deploy.sh + + deploy.sh --dev + + deploy.sh --dev --noinput orchestra + + EOF +} + + +function main () { + # Input validation + opts=$(getopt -o n:dr:h -l noinput:,dev,repo:,help -- "$@") || exit 1 + set -- $opts + + dev= + noinput= + user=$(whoami) + repo='https://github.com/ribaguifi/django-orchestra.git' + brepo= + project_name="panel" + bproject_name= + + while [ $# -gt 0 ]; do + case $1 in + -n|--noinput) user="${2:1:${#2}-2}"; noinput='--noinput'; shift ;; + -r|--repo) repo="${2:1:${#2}-2}"; brepo=true; shift ;; + -d|--dev) dev=true; ;; + -p|--project_name) project_name="${2:1:${#2}-2}"; bproject_name=true; shift ;; + -h|--help) print_help; exit 0 ;; + (--) shift; break;; + (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift + done + unset OPTIND + unset opt + + if [[ ! $noinput ]]; then + if [[ $(whoami) == 'root' ]]; then + echo -e "\nErr. Interactive script should run as a regular user\n" >&2 + exit 2 + fi + run () { echo " ${bold}\$ ${@}${normal}"; ${@}; } + surun () { echo " ${bold}\$ ${@}${normal}"; ${@}; } + else + if [[ $(whoami) != 'root' ]]; then + echo -e "\nErr. --noinput should run as root\n" >&2 + exit 3 + elif ! id $user &> /dev/null; then + echo -e "\nErr. User $2 does not exist\n" >&2 + exit 4 + fi + run () { echo " ${bold}\$ ${@}${normal}"; ${@}; } + surun () { echo " ${bold}\$ su $user -c \"${@}\"${normal}"; su $user -c "${@}"; } + fi + if [[ ! $dev && $brepo ]]; then + echo -e "\nErr. --repo only makes sense with --dev\n" >&2 + exit 5 + fi + + sudo true + if [[ ! $noinput && ! $bproject_name ]]; then + while true; do + read -p "Enter a project name [panel]: " project_name + if [[ ! "$project_name" ]]; then + project_name="panel" + break + elif [[ ! $(echo "$project_name" | grep '^[_a-zA-Z]\w*$') ]]; then + if [[ ! $(echo "$project_name" | grep '^[_a-zA-Z]') ]]; then + message='make sure the name begins with a letter or underscore' + else + message='use only numbers, letters and underscores' + fi + echo "'$project_name' is not a valid %s name. Please $message." + else + break + fi + done + fi + + task=cronbeat + if [[ ! $noinput ]]; then + while true; do + read -p "Do you want to use celery or cronbeat (orchestra.contrib.tasks) for task execution [cronbeat]? " task + case $task in + 'celery' ) task=celery; break;; + 'cronbeat' ) task=cronbeat; break;; + '' ) task=cronbeat; break;; + * ) echo "Please answer celery or cronbeat.";; + esac + done + fi + + home=$(eval echo ~$user) + cd $home + + install_orchestra "$dev" $home $repo + if [[ ! -e $project_name ]]; then + surun "orchestra-admin startproject $project_name" + else + echo "Not deploying, $project_name already exists." + fi + cd $project_name + setup_database "$dev" "$noinput" + + if [[ $noinput ]]; then + create_orchestra_superuser $user $user@localhost orchestra + fi + + if [[ "$task" == "celery" ]]; then + run sudo apt-get install rabbitmq-server + run sudo python3 -W ignore manage.py setupcelery --username $user + else + surun "python3 -W ignore manage.py setupcronbeat" + surun "python3 -W ignore manage.py syncperiodictasks" + fi + + run sudo python3 -W ignore manage.py setuplog --noinput + + surun "python3 -W ignore manage.py collectstatic --noinput" + run sudo apt-get install nginx-full uwsgi uwsgi-plugin-python3 + run sudo python3 -W ignore manage.py setupnginx --user $user $noinput + run sudo python3 -W ignore manage.py restartservices + run sudo python3 -W ignore manage.py startservices + surun "python3 -W ignore manage.py check --deploy" + + + ip_addr=$(ip addr show eth0 | grep 'inet ' | sed -r "s/.*inet ([^\s]*).*/\1/" | cut -d'/' -f1) + if [[ ! $ip_addr ]]; then + ip_addr=127.0.0.1 + fi + + # Configure settings file into debug mode + if [[ $dev ]]; then + sed -i \ + -e "s/^\s*#\s*'debug_toolbar',/ 'debug_toolbar',/" \ + -e "s/^\s*#\s*'django_nose',/ 'django_nose',/" $project_name/settings.py + if [[ ! $(grep '^INTERNAL_IPS\s*=' $project_name/settings.py) ]]; then + echo "INTERNAL_IPS = ('$ip_addr',)" >> $project_name/settings.py + fi + fi + + test_orchestra $user $ip_addr +} + +# Wrap it all on a function to avoid partial executions when running through wget/curl +main $@ From c386b10bc80fd960b4b1b8352962f3d4aa8c6195 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 16:47:26 +0100 Subject: [PATCH 06/22] Fixed --- scripts/containers/orchestra-deploy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index a936f5dd..1b936e80 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -46,7 +46,7 @@ function install_orchestra () { if [[ $dev ]]; then # Install from source - python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][0]);") + python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][1]);") if [[ -d $python_path/orchestra ]]; then run sudo rm -fr $python_path/orchestra fi @@ -76,7 +76,7 @@ function install_orchestra () { # Install from pip run sudo orchestra-admin install_requirements #run sudo pip3 install http://git.io/django-orchestra-dev - run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git + run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git#egg=django-orchestra fi } @@ -89,7 +89,7 @@ function setup_database () { if [[ $dev ]]; then # Speeding up tests, don't do this in production! . /usr/share/postgresql-common/init.d-functions - pg_version=$(psql --version | head -n1 | sed -r "s/^.*\s([0-9]+\.[0-9]+).*/\1/") + pg_version=$(psql --version | head -n1 | awk '{print $3}' | sed -e "s,\..*,,") sudo sed -i \ -e "s/^#fsync =\s*.*/fsync = off/" \ -e "s/^#full_page_writes =\s*.*/full_page_writes = off/" \ From a4c3b002053c9900ad0556e652118a4422a7021e Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 16:47:54 +0100 Subject: [PATCH 07/22] Fixed comand line of setuppostgress --- .../management/commands/setuppostgres.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index bcd14424..ff03fdf5 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -22,29 +22,53 @@ class Command(BaseCommand): if defaults['ENGINE'] != 'django.db.backends.postgresql_psycopg2': defaults = {} - self.option_list = BaseCommand.option_list + ( - make_option('--db_name', dest='db_name', - default=defaults.get('DB_NAME', 'orchestra'), - help='Specifies the database to create.'), - make_option('--db_user', dest='db_user', + def add_arguments(self, parser): + parser.add_argument( + '--db_name', + dest='db_name', + default=defaults.get('DB_NAME', 'orchestra2'), + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_user', + dest='db_user', default=defaults.get('DB_USER', 'orchestra'), - help='Specifies the database to create.'), - make_option('--db_password', dest='db_password', + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_password', + dest='db_password', default=defaults.get('PASSWORD', ''), - help='Specifies the database password, random if not specified.'), - make_option('--db_host', dest='db_host', + help='Specifies the database password, random if not specified.', + type=str + ) + parser.add_argument( + '--db_host', + dest='db_host', default=defaults.get('HOST', 'localhost'), - help='Specifies the database to create.'), - make_option('--db_port', dest='db_port', + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_port', + dest='db_port', default=defaults.get('PORT', '5432'), - help='Specifies the database to create.'), - make_option('--noinput', action='store_false', dest='interactive', default=True, + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--noinput', + action='store_false', + dest='interactive', + default=True, help='Tells Django to NOT prompt the user for input of any kind. ' 'You must use --username with --noinput, and must contain the ' - 'cleeryd process owner, which is the user how will perform tincd updates'), + 'cleeryd process owner, which is the user how will perform tincd updates', + type=str ) - option_list = BaseCommand.option_list help = 'Setup PostgreSQL database.' def run_postgres(self, cmd, *args, **kwargs): From 38275847d9d434d256b5ebaba264f059efbf9837 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 19:57:42 +0100 Subject: [PATCH 08/22] fixing results --- orchestra/management/commands/setuppostgres.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index ff03fdf5..3b552de6 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -26,7 +26,7 @@ class Command(BaseCommand): parser.add_argument( '--db_name', dest='db_name', - default=defaults.get('DB_NAME', 'orchestra2'), + default=defaults.get('DB_NAME', 'orchestra'), help='Specifies the database to create.', type=str ) @@ -106,7 +106,7 @@ class Command(BaseCommand): self.stdout.write(msg % context) else: raise CommandError("Postgres user '%(db_user)s' already exists and " - "--db_pass has not been provided." % context) + "--db_password has not been provided." % context) else: context['db_password'] = context['default_db_password'] msg = "Created new Postgres user '%(db_user)s' with password '%(db_password)s'" From 5b4b7310e659853d5eba36a840c8ff46932455cc Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 19:58:48 +0100 Subject: [PATCH 09/22] documentation ribaguifi style instalation --- .env.example | 5 + INSTALL_RIBAGUIFI_STYLE.md | 67 +++++ install_manually.md | 2 +- .../conf/ribaguifi_template/locale/.gitignore | 0 orchestra/conf/ribaguifi_template/manage.py | 13 + .../conf/ribaguifi_template/media/.gitignore | 0 .../project_name/__init__.py | 0 .../project_name/settings.py | 257 ++++++++++++++++++ .../ribaguifi_template/project_name/urls.py | 6 + .../ribaguifi_template/project_name/wsgi.py | 14 + total_requirements.txt | 39 +++ 11 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 INSTALL_RIBAGUIFI_STYLE.md create mode 100644 orchestra/conf/ribaguifi_template/locale/.gitignore create mode 100755 orchestra/conf/ribaguifi_template/manage.py create mode 100644 orchestra/conf/ribaguifi_template/media/.gitignore create mode 100644 orchestra/conf/ribaguifi_template/project_name/__init__.py create mode 100644 orchestra/conf/ribaguifi_template/project_name/settings.py create mode 100644 orchestra/conf/ribaguifi_template/project_name/urls.py create mode 100644 orchestra/conf/ribaguifi_template/project_name/wsgi.py create mode 100644 total_requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..15bbe394 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +SECRET_KEY=k_=*vfue(^campsl63)7w5m&cu9u4o4-!vaw94qzyrymyv0hgg +DEBUG=True +ALLOWED_HOSTS=.localhost,127.0.0.1 +DATABASE_URL=postgres://USER:PASSWORD@HOST:PORT/NAME +STATIC_ROOT=PATH_TO_STATIC_ROOT diff --git a/INSTALL_RIBAGUIFI_STYLE.md b/INSTALL_RIBAGUIFI_STYLE.md new file mode 100644 index 00000000..45492f22 --- /dev/null +++ b/INSTALL_RIBAGUIFI_STYLE.md @@ -0,0 +1,67 @@ +We need have python3.6 + +#Install Packages +```bash +apt=( + bind9utils + ca-certificates + gettext + libcrack2-dev + libxml2-dev + libxslt1-dev + ssh-client + wget + xvfb + zlib1g-dev + git + iceweasel + dnsutils + postgresql-contrib +) +sudo apt-get install --no-install-recommends -y ${apt[@]} +``` + +Clone this repository +```bash +git clone https://github.com/ribaguifi/django-orchestra +``` + +Prepare env and install requirements +```bash +cd django-orchestra +python3.6 -m venv env +source env/bin/activate +pip3 install --upgrade pip +pip3 install -r total_requirements.txt +pip3 install -e . +``` + +Configure project using environment file (you can use provided example as quickstart): +```bash +cp .env.example .env +``` + +Prepare your Postgres database (create database, user and grant permissions): +```sql +CREATE DATABASE myproject; +CREATE USER myuser WITH PASSWORD 'password'; +GRANT ALL PRIVILEGES ON DATABASE myproject TO myuser; +``` + +Prepare a new project: + +```bash +django-admin.py startproject PROJECT_NAME --template="orchestra/conf/ribaguifi_template" +``` + +Run migrations: +```bash +python3 manage.py migrate +``` + +(Optional) You can start a Django development server to check that everything is ok. +```bash +python3 manage.py runserver +``` + +Open [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your browser. diff --git a/install_manually.md b/install_manually.md index a2747041..51d8c660 100644 --- a/install_manually.md +++ b/install_manually.md @@ -93,7 +93,7 @@ Remember create a database for your project and give permitions for the correct ``` psql -U postgres psql (12.4) -Digite «help» para obtener ayuda. +Digite «help». postgres=# CREATE database orchesta; postgres=# CREATE USER orchesta WITH PASSWORD 'orquesta'; diff --git a/orchestra/conf/ribaguifi_template/locale/.gitignore b/orchestra/conf/ribaguifi_template/locale/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/ribaguifi_template/manage.py b/orchestra/conf/ribaguifi_template/manage.py new file mode 100755 index 00000000..ee4b9652 --- /dev/null +++ b/orchestra/conf/ribaguifi_template/manage.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import os +import sys + + +if __name__ == "__main__": + if sys.version_info < (3, 3): + cmd = ' '.join(sys.argv) + sys.stderr.write("Sorry, Orchestra requires at least Python 3.3, try with:\n$ python3 %s\n" % cmd) + sys.exit(1) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") + from django.core.management import execute_from_command_line + execute_from_command_line(sys.argv) diff --git a/orchestra/conf/ribaguifi_template/media/.gitignore b/orchestra/conf/ribaguifi_template/media/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/ribaguifi_template/project_name/__init__.py b/orchestra/conf/ribaguifi_template/project_name/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/conf/ribaguifi_template/project_name/settings.py b/orchestra/conf/ribaguifi_template/project_name/settings.py new file mode 100644 index 00000000..ee1a9512 --- /dev/null +++ b/orchestra/conf/ribaguifi_template/project_name/settings.py @@ -0,0 +1,257 @@ +""" +Django settings for {{ project_name }} project. + +Generated by 'django-admin startproject' using Django {{ django_version }}. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/ +""" + +import os +from decouple import config, Csv +from dj_database_url import parse as db_url + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '{{ secret_key }}' +# SECRET_KEY = config('SECRET_KEY') + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = config('DEBUG', default=False, cast=bool) +ALLOWED_HOSTS = config('ALLOWED_HOSTS', default=[], cast=Csv()) + + +# Application definition + +INSTALLED_APPS = [ + # django-orchestra apps + 'orchestra', + 'orchestra.contrib.accounts', + 'orchestra.contrib.systemusers', + 'orchestra.contrib.contacts', + 'orchestra.contrib.orchestration', + 'orchestra.contrib.bills', + 'orchestra.contrib.payments', + 'orchestra.contrib.tasks', + 'orchestra.contrib.mailer', + 'orchestra.contrib.history', + 'orchestra.contrib.issues', + 'orchestra.contrib.services', + 'orchestra.contrib.plans', + 'orchestra.contrib.orders', + 'orchestra.contrib.domains', + 'orchestra.contrib.mailboxes', + 'orchestra.contrib.lists', + 'orchestra.contrib.webapps', + 'orchestra.contrib.websites', + 'orchestra.contrib.letsencrypt', + 'orchestra.contrib.databases', + 'orchestra.contrib.vps', + 'orchestra.contrib.saas', + 'orchestra.contrib.miscellaneous', + + # Third-party apps + 'django_extensions', + 'djcelery', + 'fluent_dashboard', + 'admin_tools', + 'admin_tools.theming', + 'admin_tools.menu', + 'admin_tools.dashboard', + 'rest_framework', + 'rest_framework.authtoken', + 'passlib.ext.django', + 'django_countries', +# 'debug_toolbar', + + # Django.contrib + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin.apps.SimpleAdminConfig', + + # Last to load + 'orchestra.contrib.resources', + 'orchestra.contrib.settings', +# 'django_nose', +] + + +ROOT_URLCONF = '{{ project_name }}.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'orchestra.core.context_processors.site', + ], + 'loaders': [ + 'admin_tools.template_loaders.Loader', + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', + ], + }, + }, +] + + +WSGI_APPLICATION = '{{ project_name }}.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#databases + +DATABASES = { + 'default': config( + 'DATABASE_URL', + default='sqlite:///' + os.path.join(BASE_DIR, 'db.sqlite3'), + cast=db_url + ) +} + + +# Internationalization +# https://docs.djangoproject.com/en/{{ docs_version }}/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + + +try: + TIME_ZONE = open('/etc/timezone', 'r').read().strip() +except IOError: + TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/{{ docs_version }}/howto/static-files/ + +STATIC_URL = '/static/' + + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = os.path.join(BASE_DIR, 'static') + +# Absolute filesystem path to the directory that will hold user-uploaded files. +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + + +# Path used for database translations files +LOCALE_PATHS = ( + os.path.join(BASE_DIR, 'locale'), +) + +ORCHESTRA_SITE_NAME = '{{ project_name }}' + + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + # 'django.middleware.locale.LocaleMiddleware' + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'orchestra.core.caches.RequestCacheMiddleware', + # also handles transations, ATOMIC_REQUESTS does not wrap middlewares + 'orchestra.contrib.orchestration.middlewares.OperationsMiddleware', +) + + +AUTH_USER_MODEL = 'accounts.Account' + + +AUTHENTICATION_BACKENDS = [ + 'orchestra.permissions.auth.OrchestraPermissionBackend', + 'django.contrib.auth.backends.ModelBackend', +] + + +EMAIL_BACKEND = 'orchestra.contrib.mailer.backends.EmailBackend' + + +# Needed for Bulk operations +DATA_UPLOAD_MAX_NUMBER_FIELDS = None + + +################################# +## 3RD PARTY APPS CONIGURATION ## +################################# + +# Admin Tools +ADMIN_TOOLS_MENU = 'orchestra.admin.menu.OrchestraMenu' + +# Fluent dashboard +ADMIN_TOOLS_INDEX_DASHBOARD = 'orchestra.admin.dashboard.OrchestraIndexDashboard' +FLUENT_DASHBOARD_ICON_THEME = '../orchestra/icons' + + +# Django-celery +import djcelery +djcelery.setup_loader() +CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' + + +# rest_framework +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': ( + 'orchestra.permissions.api.OrchestraPermissionBackend', + ), + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.TokenAuthentication', + ), + 'DEFAULT_FILTER_BACKENDS': ( + ('rest_framework.filters.DjangoFilterBackend',) + ), +} + + +# Use a UNIX compatible hash +PASSLIB_CONFIG = ( + "[passlib]\n" + "schemes = sha512_crypt, django_pbkdf2_sha256, django_pbkdf2_sha1, " + " django_bcrypt, django_bcrypt_sha256, django_salted_sha1, des_crypt, " + " django_salted_md5, django_des_crypt, hex_md5, bcrypt, phpass\n" + "default = sha512_crypt\n" + "deprecated = django_pbkdf2_sha1, django_salted_sha1, django_salted_md5, " + " django_des_crypt, des_crypt, hex_md5\n" + "all__vary_rounds = 0.05\n" + "django_pbkdf2_sha256__min_rounds = 10000\n" + "sha512_crypt__min_rounds = 80000\n" + "staff__django_pbkdf2_sha256__default_rounds = 12500\n" + "staff__sha512_crypt__default_rounds = 100000\n" + "superuser__django_pbkdf2_sha256__default_rounds = 15000\n" + "superuser__sha512_crypt__default_rounds = 120000\n" +) + + +SHELL_PLUS_PRE_IMPORTS = ( + ('orchestra.contrib.orchestration.managers', ('orchestrate',)), +) diff --git a/orchestra/conf/ribaguifi_template/project_name/urls.py b/orchestra/conf/ribaguifi_template/project_name/urls.py new file mode 100644 index 00000000..3ae27421 --- /dev/null +++ b/orchestra/conf/ribaguifi_template/project_name/urls.py @@ -0,0 +1,6 @@ +from django.conf.urls import include, url + + +urlpatterns = [ + url(r'', include('orchestra.urls')), +] diff --git a/orchestra/conf/ribaguifi_template/project_name/wsgi.py b/orchestra/conf/ribaguifi_template/project_name/wsgi.py new file mode 100644 index 00000000..94d60c8c --- /dev/null +++ b/orchestra/conf/ribaguifi_template/project_name/wsgi.py @@ -0,0 +1,14 @@ +""" +WSGI config for {{ project_name }} project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/ +""" + +import os +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings") + +from django.core.wsgi import get_wsgi_application +application = get_wsgi_application() diff --git a/total_requirements.txt b/total_requirements.txt new file mode 100644 index 00000000..98f23183 --- /dev/null +++ b/total_requirements.txt @@ -0,0 +1,39 @@ +Django==1.10.5 +django-fluent-dashboard==0.6.1 +django-admin-tools==0.8.0 +django-extensions==1.7.4 +django-celery==3.1.17 +celery==3.1.23 +kombu==3.0.35 +billiard==3.3.0.23 +Markdown==2.4 +djangorestframework==3.4.7 +ecdsa==0.11 +Pygments==1.6 +django-filter==0.15.2 +jsonfield==0.9.22 +python-dateutil==2.2 +django-iban==0.3.0 +requests +phonenumbers +django-countries +django-localflavor +amqp +anyjson +pytz +cracklib +lxml==3.3.5 +selenium +xvfbwrapper +freezegun +coverage +flake8 +django-debug-toolbar==1.3.0 +django-nose==1.4.4 +sqlparse +pyinotify +PyMySQL +dj_database_url==0.5.0 +psycopg2-binary +python-decouple +https://github.com/glic3rinu/passlib/archive/master.zip From 0cde41042f0d68f5ae6fed36dfc4d5affcbf1b82 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 28 Jan 2021 20:07:17 +0100 Subject: [PATCH 10/22] wkhtmltopdf --- INSTALL_RIBAGUIFI_STYLE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/INSTALL_RIBAGUIFI_STYLE.md b/INSTALL_RIBAGUIFI_STYLE.md index 45492f22..ee821040 100644 --- a/INSTALL_RIBAGUIFI_STYLE.md +++ b/INSTALL_RIBAGUIFI_STYLE.md @@ -21,6 +21,9 @@ apt=( sudo apt-get install --no-install-recommends -y ${apt[@]} ``` +It is necessary install *wkhtmltopdf* +You can install it from https://wkhtmltopdf.org/downloads.html + Clone this repository ```bash git clone https://github.com/ribaguifi/django-orchestra From 24e75bc07ff95835a0f4bf61390759bd9de5b86a Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 29 Jan 2021 11:44:10 +0100 Subject: [PATCH 11/22] fixed pg --- orchestra/management/commands/setuppostgres.py | 8 ++++---- scripts/containers/orchestra-deploy | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index 3b552de6..4dbaeb26 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -79,11 +79,11 @@ class Command(BaseCommand): interactive = options.get('interactive') db_password = options.get('db_password') context = { - 'db_name': options.get('db_name'), - 'db_user': options.get('db_user'), + 'db_name': options.get('db_name', 'orchestra'), + 'db_user': options.get('db_user', 'orchestra'), 'db_password': db_password, - 'db_host': options.get('db_host'), - 'db_port': options.get('db_port'), + 'db_host': options.get('db_host', '127.0.0.1'), + 'db_port': options.get('db_port', '5432'), 'default_db_password': db_password or random_ascii(10), } diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index 1b936e80..c5203e1a 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -69,8 +69,9 @@ function install_orchestra () { if [[ -L /usr/local/bin/orchestra-admin || -f /usr/local/bin/orchestra-admin ]]; then run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} fi - run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/ - run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-beat /usr/local/bin/ + run sudo pip3 install -e $home/django-orchestra + #run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/ + #run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-beat /usr/local/bin/ run sudo orchestra-admin install_requirements --testing else # Install from pip @@ -84,7 +85,8 @@ function install_orchestra () { function setup_database () { dev=$1 noinput=$2 - run sudo apt-get install -y postgresql python3-psycopg2 + run sudo apt-get install -y postgresql + run sudo pip install psycopg2 # Setup Database if [[ $dev ]]; then # Speeding up tests, don't do this in production! From 2619a5041090bb4dd99aebf81bbe5089e852ce34 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 29 Jan 2021 13:04:22 +0100 Subject: [PATCH 12/22] change branch --- scripts/containers/orchestra-deploy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index c5203e1a..58c6dadd 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -62,6 +62,8 @@ function install_orchestra () { # Finishing partial installation surun "export GIT_DIR=$home/django-orchestra/.git; git pull" } + # TODO remove this line when it branch is merged to master + surun "cd $home/django-orchestra/; git checkout docker" echo $home/django-orchestra/ | sudo tee "$python_path/orchestra.pth" else echo "You may want to execute 'git pull origin master'?" From 6450d0d7494893a61d998e186af2d3a95dc83cf6 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 29 Jan 2021 14:11:54 +0100 Subject: [PATCH 13/22] clean code for work in master --- INSTALLDEV.md | 7 ++++--- scripts/containers/Dockerfile | 2 +- scripts/containers/orchestra-deploy | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/INSTALLDEV.md b/INSTALLDEV.md index 96420d12..3b9f78ac 100644 --- a/INSTALLDEV.md +++ b/INSTALLDEV.md @@ -11,7 +11,7 @@ If you are planing to do some development you may want to consider doing it unde 2. Build a new image, create and start a container ```bash - curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/Dockerfile > /tmp/Dockerfile + curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/Dockerfile > /tmp/Dockerfile docker build -t orchestra /tmp/ docker create --name orchestra -i -t -u orchestra -w /home/orchestra orchestra bash docker start orchestra @@ -21,12 +21,13 @@ If you are planing to do some development you may want to consider doing it unde 3. Deploy django-orchestra development environment, inside the container ```bash - bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/orchestra-deploy ) --dev ``` 3. Nginx should be serving on port 80, but Django's development server can be used as well: ```bash cd panel + python3 manage.py migrate python3 manage.py runserver 0.0.0.0:8888 ``` @@ -34,5 +35,5 @@ If you are planing to do some development you may want to consider doing it unde 5. To upgrade to current master just re-run the deploy script ```bash git pull origin master - bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/scripts/containers/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/orchestra-deploy ) --dev ``` diff --git a/scripts/containers/Dockerfile b/scripts/containers/Dockerfile index 35c92639..d06a0af1 100644 --- a/scripts/containers/Dockerfile +++ b/scripts/containers/Dockerfile @@ -132,7 +132,7 @@ RUN set -ex; \ RUN apt-get -y update && apt-get install -y curl sudo -RUN export TERM=xterm; curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/docker/orchestra/bin/orchestra-admin | bash -s install_requirements +RUN export TERM=xterm; curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/orchestra/bin/orchestra-admin | bash -s install_requirements RUN apt-get clean diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index 58c6dadd..c5203e1a 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -62,8 +62,6 @@ function install_orchestra () { # Finishing partial installation surun "export GIT_DIR=$home/django-orchestra/.git; git pull" } - # TODO remove this line when it branch is merged to master - surun "cd $home/django-orchestra/; git checkout docker" echo $home/django-orchestra/ | sudo tee "$python_path/orchestra.pth" else echo "You may want to execute 'git pull origin master'?" From d0050f81b70c19116db9992d8c1dfc07c50bdeea Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 13:18:03 +0100 Subject: [PATCH 14/22] clean orchestra-deploy --- scripts/containers/orchestra-deploy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index c5203e1a..6ac68c1c 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -50,8 +50,7 @@ function install_orchestra () { if [[ -d $python_path/orchestra ]]; then run sudo rm -fr $python_path/orchestra fi - #orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null) - orch_version='' + orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null) if [[ ! $orch_version ]]; then # First Orchestra installation run sudo mkdir -p /usr/share/man/man1 @@ -70,13 +69,10 @@ function install_orchestra () { run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} fi run sudo pip3 install -e $home/django-orchestra - #run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/ - #run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-beat /usr/local/bin/ run sudo orchestra-admin install_requirements --testing else # Install from pip run sudo orchestra-admin install_requirements - #run sudo pip3 install http://git.io/django-orchestra-dev run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git#egg=django-orchestra fi } From 7c62092faa457e040f3d56f79beeb911b63e467d Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 13:43:03 +0100 Subject: [PATCH 15/22] fixed deploy --- scripts/containers/orchestra-deploy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy index 6ac68c1c..45fdde20 100644 --- a/scripts/containers/orchestra-deploy +++ b/scripts/containers/orchestra-deploy @@ -50,7 +50,7 @@ function install_orchestra () { if [[ -d $python_path/orchestra ]]; then run sudo rm -fr $python_path/orchestra fi - orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null) + orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null || echo '') if [[ ! $orch_version ]]; then # First Orchestra installation run sudo mkdir -p /usr/share/man/man1 From 78db4fb8d5070e8ac52fc5493db85f3ee0494a0a Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 13:55:27 +0100 Subject: [PATCH 16/22] fixed --- .../management/commands/setuppostgres.py | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index 4dbaeb26..ff365a75 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -11,65 +11,65 @@ from orchestra.utils.sys import run, check_root class Command(BaseCommand): + help = 'Setup PostgreSQL database.' + def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) # Get defaults from settings, if exists try: - defaults = settings.DATABASES['default'] + self.defaults = settings.DATABASES['default'] except (AttributeError, KeyError): defaults = {} else: - if defaults['ENGINE'] != 'django.db.backends.postgresql_psycopg2': - defaults = {} + if self.defaults['ENGINE'] != 'django.db.backends.postgresql_psycopg2': + self.defaults = {} - def add_arguments(self, parser): - parser.add_argument( - '--db_name', - dest='db_name', - default=defaults.get('DB_NAME', 'orchestra'), - help='Specifies the database to create.', - type=str - ) - parser.add_argument( - '--db_user', - dest='db_user', - default=defaults.get('DB_USER', 'orchestra'), - help='Specifies the database to create.', - type=str - ) - parser.add_argument( - '--db_password', - dest='db_password', - default=defaults.get('PASSWORD', ''), - help='Specifies the database password, random if not specified.', - type=str - ) - parser.add_argument( - '--db_host', - dest='db_host', - default=defaults.get('HOST', 'localhost'), - help='Specifies the database to create.', - type=str - ) - parser.add_argument( - '--db_port', - dest='db_port', - default=defaults.get('PORT', '5432'), - help='Specifies the database to create.', - type=str - ) - parser.add_argument( - '--noinput', - action='store_false', - dest='interactive', - default=True, - help='Tells Django to NOT prompt the user for input of any kind. ' - 'You must use --username with --noinput, and must contain the ' - 'cleeryd process owner, which is the user how will perform tincd updates', - type=str - ) - - help = 'Setup PostgreSQL database.' + def add_arguments(self, parser): + parser.add_argument( + '--db_name', + dest='db_name', + default=self.defaults.get('DB_NAME', 'orchestra'), + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_user', + dest='db_user', + default=self.defaults.get('DB_USER', 'orchestra'), + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_password', + dest='db_password', + default=self.defaults.get('PASSWORD', ''), + help='Specifies the database password, random if not specified.', + type=str + ) + parser.add_argument( + '--db_host', + dest='db_host', + default=slef.defaults.get('HOST', 'localhost'), + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--db_port', + dest='db_port', + default=self.defaults.get('PORT', '5432'), + help='Specifies the database to create.', + type=str + ) + parser.add_argument( + '--noinput', + action='store_false', + dest='interactive', + default=True, + help='Tells Django to NOT prompt the user for input of any kind. ' + 'You must use --username with --noinput, and must contain the ' + 'cleeryd process owner, which is the user how will perform tincd updates', + type=str + ) def run_postgres(self, cmd, *args, **kwargs): return run('su postgres -c "psql -c \\"%s\\""' % cmd, *args, **kwargs) From 8da89ae22afc01f332bfdfbe400772f1c0c215a4 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 14:17:18 +0100 Subject: [PATCH 17/22] fixing commands we need --- orchestra/management/commands/setupnginx.py | 155 ++++++++++++------ .../management/commands/setuppostgres.py | 17 +- 2 files changed, 117 insertions(+), 55 deletions(-) diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index 6ee9b911..98ca84f7 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -11,54 +11,117 @@ from orchestra.utils.sys import run, check_root, confirm class Command(BaseCommand): + help = 'Configures nginx + uwsgi to run with your Orchestra instance.' + def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + ( - make_option('--cert', dest='cert', default='', - help='Nginx SSL certificate, one will be created by default.'), - make_option('--cert-key', dest='cert_key', default='', - help='Nginx SSL certificate key.'), - - make_option('--cert-path', dest='cert_path', - default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'), - help='Nginx SSL certificate, one will be created by default.'), - make_option('--cert-key-path', dest='cert_key_path', - default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'), - help='Nginx SSL certificate key.'), - # Cert options - make_option('--cert-override', dest='cert_override', action='store_true', - default=False, help='Force override cert and keys if exists.'), - make_option('--cert-country', dest='cert_country', default='ES', - help='Certificate Distinguished Name Country.'), - make_option('--cert-state', dest='cert_state', default='Spain', - help='Certificate Distinguished Name STATE.'), - make_option('--cert-locality', dest='cert_locality', default='Barcelona', - help='Certificate Distinguished Name Country.'), - make_option('--cert-org_name', dest='cert_org_name', default='Orchestra', - help='Certificate Distinguished Name Organization Name.'), - make_option('--cert-org_unit', dest='cert_org_unit', default='DevOps', - help='Certificate Distinguished Name Organization Unity.'), - make_option('--cert-email', dest='cert_email', default='orchestra@orchestra.lan', - help='Certificate Distinguished Name Email Address.'), - make_option('--cert-common_name', dest='cert_common_name', default=None, - help='Certificate Distinguished Name Common Name.'), - - make_option('--server-name', dest='server_name', default='', - help='Nginx SSL certificate key.'), - make_option('--user', dest='user', default='', - help='uWSGI daemon user.'), - make_option('--group', dest='group', default='', - help='uWSGI daemon group.'), - make_option('--processes', dest='processes', default=4, - help='uWSGI number of processes.'), - make_option('--noinput', action='store_false', dest='interactive', default=True, - help='Tells Django to NOT prompt the user for input of any kind. ' - 'You must use --username with --noinput, and must contain the ' - 'cleeryd process owner, which is the user how will perform tincd updates'), - ) - - option_list = BaseCommand.option_list - help = 'Configures nginx + uwsgi to run with your Orchestra instance.' + + def add_arguments(self, parser): + parser.add_argument( + '--cert', + dest='cert', + default='', + help='Nginx SSL certificate, one will be created by default.', + ) + parser.add_argument( + '--cert-key', + dest='cert_key', + default='', + help='Nginx SSL certificate key.' + ) + parser.add_argument( + '--cert-path', + dest='cert_path', + default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'), + help='Nginx SSL certificate, one will be created by default.' + ) + parser.add_argument( + '--cert-key-path', + dest='cert_key_path', + default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'), + help='Nginx SSL certificate key.' + ) + parser.add_argument( + '--cert-override', + dest='cert_override', + action='store_true', + default=False, help='Force override cert and keys if exists.' + ) + parser.add_argument( + '--cert-country', + dest='cert_country', + default='ES', + help='Certificate Distinguished Name Country.' + ) + parser.add_argument( + '--cert-state', + dest='cert_state', + default='Spain', + help='Certificate Distinguished Name STATE.' + ) + parser.add_argument( + '--cert-locality', + dest='cert_locality', + default='Barcelona', + help='Certificate Distinguished Name Country.' + ) + parser.add_argument( + '--cert-org_name', + dest='cert_org_name', + default='Orchestra', + help='Certificate Distinguished Name Organization Name.' + ) + parser.add_argument( + '--cert-org_unit', + dest='cert_org_unit', + default='DevOps', + help='Certificate Distinguished Name Organization Unity.' + ) + parser.add_argument( + '--cert-email', + dest='cert_email', + default='orchestra@orchestra.lan', + help='Certificate Distinguished Name Email Address.' + ) + parser.add_argument( + '--cert-common_name', + dest='cert_common_name', + default=None, + help='Certificate Distinguished Name Common Name.' + ) + parser.add_argument( + '--server-name', + dest='server_name', + default='', + help='Nginx SSL certificate key.' + ) + parser.add_argument( + '--user', + dest='user', + default='', + help='uWSGI daemon user.' + ) + parser.add_argument( + '--group', + dest='group', + default='', + help='uWSGI daemon group.' + ) + parser.add_argument( + '--processes', + dest='processes', + default=4, + help='uWSGI number of processes.' + ) + parser.add_argument( + '--noinput', + action='store_false', + dest='interactive', + default=True, + help='''Tells Django to NOT prompt the user for input of any kind. + You must use --username with --noinput, and must contain the + cleeryd process owner, which is the user how will perform tincd updates''' + ) def generate_certificate(self, **options): override = options.get('cert_override') diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index ff365a75..493b42bd 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -49,7 +49,7 @@ class Command(BaseCommand): parser.add_argument( '--db_host', dest='db_host', - default=slef.defaults.get('HOST', 'localhost'), + default=self.defaults.get('HOST', 'localhost'), help='Specifies the database to create.', type=str ) @@ -65,10 +65,9 @@ class Command(BaseCommand): action='store_false', dest='interactive', default=True, - help='Tells Django to NOT prompt the user for input of any kind. ' - 'You must use --username with --noinput, and must contain the ' - 'cleeryd process owner, which is the user how will perform tincd updates', - type=str + help='''Tells Django to NOT prompt the user for input of any kind. + You must use --username with --noinput, and must contain the + cleeryd process owner, which is the user how will perform tincd updates''' ) def run_postgres(self, cmd, *args, **kwargs): @@ -79,11 +78,11 @@ class Command(BaseCommand): interactive = options.get('interactive') db_password = options.get('db_password') context = { - 'db_name': options.get('db_name', 'orchestra'), - 'db_user': options.get('db_user', 'orchestra'), + 'db_name': options.get('db_name'), + 'db_user': options.get('db_user'), 'db_password': db_password, - 'db_host': options.get('db_host', '127.0.0.1'), - 'db_port': options.get('db_port', '5432'), + 'db_host': options.get('db_host'), + 'db_port': options.get('db_port'), 'default_db_password': db_password or random_ascii(10), } From 6ce4d6b8777e0ac8c35af58054c72174775d601f Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 14:59:49 +0100 Subject: [PATCH 18/22] fixed make_options for all commands --- .../commands/postupgradeorchestra.py | 49 ++++--- orchestra/management/commands/setupcelery.py | 41 +++--- orchestra/management/commands/setupnginx.py | 1 - orchestra/management/commands/setuppostfix.py | 123 ++++++++++++------ .../management/commands/setuppostgres.py | 1 - .../management/commands/startservices.py | 15 ++- .../management/commands/upgradeorchestra.py | 31 +++-- 7 files changed, 172 insertions(+), 89 deletions(-) diff --git a/orchestra/management/commands/postupgradeorchestra.py b/orchestra/management/commands/postupgradeorchestra.py index 0e453558..992b5f90 100644 --- a/orchestra/management/commands/postupgradeorchestra.py +++ b/orchestra/management/commands/postupgradeorchestra.py @@ -1,6 +1,5 @@ import re import os -from optparse import make_option from django.core.management.base import BaseCommand @@ -19,26 +18,44 @@ def deprecate_periodic_tasks(names): class Command(BaseCommand): - def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + ( - make_option('--no-restart', action='store_false', dest='restart', default=True, - help='Only install local requirements'), - make_option('--specifics', action='store_true', dest='specifics_only', - default=False, help='Only run version specific operations'), - make_option('--no-upgrade-notes', action='store_false', default=True, - dest='print_upgrade_notes', help='Do not print specific upgrade notes'), - make_option('--from', dest='version', default=False, - help="Orchestra's version from where you are upgrading, i.e 0.0.1"), - ) - - option_list = BaseCommand.option_list help = 'Upgrades django-orchestra installation' # This command must be able to run in an environment with unsatisfied dependencies leave_locale_alone = True can_import_settings = False requires_model_validation = False - + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + + def add_arguments(self, parser): + parser.add_argument( + '--no-restart', + action='store_false', + dest='restart', + default=True, + help='Only install local requirements' + ) + parser.add_argument( + '--specifics', + action='store_true', + dest='specifics_only', + default=False, + help='Only run version specific operations' + ) + parser.add_argument( + '--no-upgrade-notes', + action='store_false', + default=True, + dest='print_upgrade_notes', + help='Do not print specific upgrade notes' + ) + parser.add_argument( + '--from', + dest='version', + default=False, + help="Orchestra's version from where you are upgrading, i.e 0.0.1" + ) + @check_root def handle(self, *args, **options): version = options.get('version') diff --git a/orchestra/management/commands/setupcelery.py b/orchestra/management/commands/setupcelery.py index c343d6e8..b2cc387a 100644 --- a/orchestra/management/commands/setupcelery.py +++ b/orchestra/management/commands/setupcelery.py @@ -1,5 +1,4 @@ import textwrap -from optparse import make_option from os import path from django.core.management.base import BaseCommand @@ -10,22 +9,34 @@ from orchestra.utils.sys import run, check_root class Command(BaseCommand): - def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + ( - make_option('--username', dest='username', default='orchestra', - help='Specifies the system user that would run celeryd.'), - make_option('--processes', dest='processes', default=2, - help='Number of celeryd processes.'), - make_option('--noinput', action='store_false', dest='interactive', default=True, - help='Tells Django to NOT prompt the user for input of any kind. ' - 'You must use --username with --noinput, and must contain the ' - 'cleleryd process owner, which is the user how will perform tincd updates'), - ) - - option_list = BaseCommand.option_list help = 'Configure Celeryd to run with your orchestra instance.' + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + + def add_arguments(self, parser): + parser.add_argument( + '--username', + dest='username', + default='orchestra', + help='Specifies the system user that would run celeryd.' + ) + parser.add_argument( + '--processes', + dest='processes', + default=2, + help='Number of celeryd processes.' + ) + parser.add_argument( + '--noinput', + action='store_false', + dest='interactive', + default=True, + help='''Tells Django to NOT prompt the user for input of any kind. + You must use --username with --noinput, and must contain the + cleleryd process owner, which is the user how will perform tincd updates''' + ) + @check_root def handle(self, *args, **options): context = { diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index 98ca84f7..db15b9ee 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -1,6 +1,5 @@ import os import textwrap -from optparse import make_option from os.path import expanduser from django.conf import settings diff --git a/orchestra/management/commands/setuppostfix.py b/orchestra/management/commands/setuppostfix.py index a3a75239..0365015f 100644 --- a/orchestra/management/commands/setuppostfix.py +++ b/orchestra/management/commands/setuppostfix.py @@ -1,52 +1,97 @@ import os -from optparse import make_option - from django.core.management.base import BaseCommand from orchestra.utils.sys import run, check_root class Command(BaseCommand): + help = 'Setup Postfix.' + def __init__(self, *args, **kwargs): super(Command, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + ( - make_option('--db_name', dest='db_name', default='orchestra', - help='Specifies the database to create.'), - make_option('--db_user', dest='db_user', default='orchestra', - help='Specifies the database to create.'), - make_option('--db_password', dest='db_password', default='orchestra', - help='Specifies the database to create.'), - make_option('--db_host', dest='db_host', default='localhost', - help='Specifies the database to create.'), - make_option('--vmail_username', dest='vmail_username', default='vmail', - help='Specifies username in the operating system (default=vmail).'), - make_option('--vmail_uid', dest='vmail_uid', default='5000', - help='UID of user (default=5000).'), - make_option('--vmail_groupname', dest='vmail_groupname', default='vmail', - help='Specifies the groupname in the operating system (default=vmail).'), - make_option('--vmail_gid', dest='vmail_gid', default='5000', - help='GID of user (default=5000).'), - make_option('--vmail_home', dest='vmail_home', default='/var/vmail', - help='$HOME of user (default=/var/vmail).'), - - make_option('--dovecot_dir', dest='dovecot_dir', default='/etc/dovecot', - help='Dovecot root directory (default=/etc/dovecot).'), - - make_option('--postfix_dir', dest='postfix_dir', default='/etc/postfix', - help='Postfix root directory (default=/etc/postfix).'), - - make_option('--amavis_dir', dest='amavis_dir', default='/etc/amavis', - help='Amavis root directory (default=/etc/amavis).'), - - make_option('--noinput', action='store_false', dest='interactive', default=True, - help='Tells Django to NOT prompt the user for input of any kind. ' - 'You must use --username with --noinput, and must contain the ' - 'cleeryd process owner, which is the user how will perform tincd updates'), - ) - - option_list = BaseCommand.option_list - help = 'Setup Postfix.' + def add_arguments(self, parser): + parser.add_argument( + '--db_name', + dest='db_name', + default='orchestra', + help='Specifies the database to create.' + ) + parser.add_argument( + '--db_user', + dest='db_user', + default='orchestra', + help='Specifies the database to create.' + ) + parser.add_argument( + '--db_password', + dest='db_password', + default='orchestra', + help='Specifies the database to create.' + ) + parser.add_argument( + '--db_host', + dest='db_host', + default='localhost', + help='Specifies the database to create.' + ) + parser.add_argument( + '--vmail_username', + dest='vmail_username', + default='vmail', + help='Specifies username in the operating system (default=vmail).' + ) + parser.add_argument( + '--vmail_uid', + dest='vmail_uid', + default='5000', + help='UID of user (default=5000).' + ) + parser.add_argument( + '--vmail_groupname', + dest='vmail_groupname', + default='vmail', + help='Specifies the groupname in the operating system (default=vmail).' + ) + parser.add_argument( + '--vmail_gid', + dest='vmail_gid', + default='5000', + help='GID of user (default=5000).' + ) + parser.add_argument( + '--vmail_home', + dest='vmail_home', + default='/var/vmail', + help='$HOME of user (default=/var/vmail).' + ) + parser.add_argument( + '--dovecot_dir', + dest='dovecot_dir', + default='/etc/dovecot', + help='Dovecot root directory (default=/etc/dovecot).' + ) + parser.add_argument( + '--postfix_dir', + dest='postfix_dir', + default='/etc/postfix', + help='Postfix root directory (default=/etc/postfix).' + ) + parser.add_argument( + '--amavis_dir', + dest='amavis_dir', + default='/etc/amavis', + help='Amavis root directory (default=/etc/amavis).' + ) + parser.add_argument( + '--noinput', + action='store_false', + dest='interactive', + default=True, + help='''Tells Django to NOT prompt the user for input of any kind. + You must use --username with --noinput, and must contain the + cleeryd process owner, which is the user how will perform tincd updates''' + ) @check_root def handle(self, *args, **options): diff --git a/orchestra/management/commands/setuppostgres.py b/orchestra/management/commands/setuppostgres.py index 493b42bd..a39ec377 100644 --- a/orchestra/management/commands/setuppostgres.py +++ b/orchestra/management/commands/setuppostgres.py @@ -1,6 +1,5 @@ import os import textwrap -from optparse import make_option from django.conf import settings from django.core.management.base import BaseCommand, CommandError diff --git a/orchestra/management/commands/startservices.py b/orchestra/management/commands/startservices.py index 6bfe9f17..55953e6a 100644 --- a/orchestra/management/commands/startservices.py +++ b/orchestra/management/commands/startservices.py @@ -1,5 +1,3 @@ -from optparse import make_option - from django.core.management.base import BaseCommand from orchestra import settings @@ -30,9 +28,16 @@ def flatten(nested, depth=0): class ManageServiceCommand(BaseCommand): def __init__(self, *args, **kwargs): super(ManageServiceCommand, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + tuple( - make_option('--no-%s' % service, action='store_false', dest=service, default=True, - help='Do not %s %s' % (self.action, service)) for service in flatten(self.services) + + + def add_arguments(self, parser): + for service in flatten(self.services): + parser.add_argument( + '--no-%s' % service, + action='store_false', + dest=service, + default=True, + help='Do not %s %s' % (self.action, service) ) @check_root diff --git a/orchestra/management/commands/upgradeorchestra.py b/orchestra/management/commands/upgradeorchestra.py index 3b7fd846..ccbdab91 100644 --- a/orchestra/management/commands/upgradeorchestra.py +++ b/orchestra/management/commands/upgradeorchestra.py @@ -3,7 +3,6 @@ import os import random import string from distutils.sysconfig import get_python_lib -from optparse import make_option from django.core.management import call_command from django.core.management.base import BaseCommand, CommandError @@ -26,20 +25,28 @@ def get_existing_pip_installation(): class Command(BaseCommand): - def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) - self.option_list = BaseCommand.option_list + ( - make_option('--pip_only', action='store_true', dest='pip_only', default=False, - help='Only run "pip install django-orchestra --upgrade"'), - make_option('--orchestra_version', dest='version', default=False, - help='Specifies what version of the Orchestra you want to install'), - ) - - option_list = BaseCommand.option_list help = "Upgrading Orchestra's installation. Desired version is accepted as argument" can_import_settings = False leave_locale_alone = True - + + def __init__(self, *args, **kwargs): + super(Command, self).__init__(*args, **kwargs) + + def add_arguments(self, parser): + parser.add_argument( + '--pip_only', + action='store_true', + dest='pip_only', + default=False, + help='Only run "pip install django-orchestra --upgrade"' + ) + parser.add_argument( + '--orchestra_version', + dest='version', + default=False, + help='Specifies what version of the Orchestra you want to install' + ) + @check_root def handle(self, *args, **options): current_version = get_version() From a236bbdf5df78633049dcd01eba640f5013d4234 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 15:06:48 +0100 Subject: [PATCH 19/22] fixed start services --- orchestra/management/commands/startservices.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orchestra/management/commands/startservices.py b/orchestra/management/commands/startservices.py index 55953e6a..d17c2c80 100644 --- a/orchestra/management/commands/startservices.py +++ b/orchestra/management/commands/startservices.py @@ -59,5 +59,4 @@ class ManageServiceCommand(BaseCommand): class Command(ManageServiceCommand): services = settings.ORCHESTRA_START_SERVICES action = 'start' - option_list = BaseCommand.option_list help = 'Start all related services. Usefull for reload configuration and files.' From 898c6882c82d0db358841ac2a35fe9acb11fdc39 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 15:09:30 +0100 Subject: [PATCH 20/22] fixed stop and restart services --- orchestra/management/commands/restartservices.py | 1 - orchestra/management/commands/stopservices.py | 1 - 2 files changed, 2 deletions(-) diff --git a/orchestra/management/commands/restartservices.py b/orchestra/management/commands/restartservices.py index 63754897..ea613566 100644 --- a/orchestra/management/commands/restartservices.py +++ b/orchestra/management/commands/restartservices.py @@ -7,5 +7,4 @@ from orchestra.management.commands.startservices import ManageServiceCommand class Command(ManageServiceCommand): services = settings.ORCHESTRA_RESTART_SERVICES action = 'restart' - option_list = BaseCommand.option_list help = 'Restart all related services. Usefull for reload configuration and files.' diff --git a/orchestra/management/commands/stopservices.py b/orchestra/management/commands/stopservices.py index ac0e6583..d0868b7a 100644 --- a/orchestra/management/commands/stopservices.py +++ b/orchestra/management/commands/stopservices.py @@ -7,5 +7,4 @@ from orchestra.management.commands.startservices import ManageServiceCommand class Command(ManageServiceCommand): services = settings.ORCHESTRA_STOP_SERVICES action = 'stop' - option_list = BaseCommand.option_list help = 'Stop all related services. Usefull for reload configuration and files.' From 883cf631e2fcc36bf310803856072ef8fc682e3a Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Sat, 30 Jan 2021 15:17:03 +0100 Subject: [PATCH 21/22] mv orchestra-deploy to deploy.sh --- INSTALLDEV.md | 4 +- scripts/containers/deploy.sh | 19 +- scripts/containers/orchestra-deploy | 306 ---------------------------- 3 files changed, 13 insertions(+), 316 deletions(-) delete mode 100644 scripts/containers/orchestra-deploy diff --git a/INSTALLDEV.md b/INSTALLDEV.md index 3b9f78ac..e8a26c73 100644 --- a/INSTALLDEV.md +++ b/INSTALLDEV.md @@ -21,7 +21,7 @@ If you are planing to do some development you may want to consider doing it unde 3. Deploy django-orchestra development environment, inside the container ```bash - bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/deploy.sh ) --dev ``` 3. Nginx should be serving on port 80, but Django's development server can be used as well: @@ -35,5 +35,5 @@ If you are planing to do some development you may want to consider doing it unde 5. To upgrade to current master just re-run the deploy script ```bash git pull origin master - bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/orchestra-deploy ) --dev + bash <( curl -L https://raw.githubusercontent.com/ribaguifi/django-orchestra/master/scripts/containers/deploy.sh ) --dev ``` diff --git a/scripts/containers/deploy.sh b/scripts/containers/deploy.sh index 755e2cdb..45fdde20 100644 --- a/scripts/containers/deploy.sh +++ b/scripts/containers/deploy.sh @@ -46,13 +46,16 @@ function install_orchestra () { if [[ $dev ]]; then # Install from source - python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][0]);") + python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][1]);") if [[ -d $python_path/orchestra ]]; then run sudo rm -fr $python_path/orchestra fi - orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null) + orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null || echo '') if [[ ! $orch_version ]]; then # First Orchestra installation + run sudo mkdir -p /usr/share/man/man1 + run sudo mkdir -p /usr/share/man/man7 + run sudo apt-get update run sudo apt-get -y install git python3-pip surun "git clone $repo $home/django-orchestra" || { # Finishing partial installation @@ -65,13 +68,12 @@ function install_orchestra () { if [[ -L /usr/local/bin/orchestra-admin || -f /usr/local/bin/orchestra-admin ]]; then run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} fi - run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/ - run sudo ln -s $home/django-orchestra/orchestra/bin/orchestra-beat /usr/local/bin/ + run sudo pip3 install -e $home/django-orchestra run sudo orchestra-admin install_requirements --testing else # Install from pip - run sudo pip3 install http://git.io/django-orchestra-dev run sudo orchestra-admin install_requirements + run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git#egg=django-orchestra fi } @@ -79,12 +81,13 @@ function install_orchestra () { function setup_database () { dev=$1 noinput=$2 - run sudo apt-get install -y postgresql python3-psycopg2 + run sudo apt-get install -y postgresql + run sudo pip install psycopg2 # Setup Database if [[ $dev ]]; then # Speeding up tests, don't do this in production! . /usr/share/postgresql-common/init.d-functions - pg_version=$(psql --version | head -n1 | sed -r "s/^.*\s([0-9]+\.[0-9]+).*/\1/") + pg_version=$(psql --version | head -n1 | awk '{print $3}' | sed -e "s,\..*,,") sudo sed -i \ -e "s/^#fsync =\s*.*/fsync = off/" \ -e "s/^#full_page_writes =\s*.*/full_page_writes = off/" \ @@ -170,7 +173,7 @@ function main () { dev= noinput= user=$(whoami) - repo='https://github.com/glic3rinu/django-orchestra.git' + repo='https://github.com/ribaguifi/django-orchestra.git' brepo= project_name="panel" bproject_name= diff --git a/scripts/containers/orchestra-deploy b/scripts/containers/orchestra-deploy deleted file mode 100644 index 45fdde20..00000000 --- a/scripts/containers/orchestra-deploy +++ /dev/null @@ -1,306 +0,0 @@ -#!/bin/bash - -set -ue - -# https://git.io/deploy-orchestra -# bash <( curl https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/scripts/containers/deploy.sh ) [--noinput username] - - -bold=$(tput -T ${TERM:-xterm} bold) -normal=$(tput -T ${TERM:-xterm} sgr0) - - -function test_orchestra () { - user=$1 - ip_addr=$2 - # Test if serving requests - echo - echo ${bold} - echo "> Checking if Orchestra is serving on https://${ip_addr}/admin/ ..." - if [[ $noinput ]]; then - echo " - username: $user" - echo " - password: orchestra${normal}" - fi - echo - if [[ $(curl -s -L -k -c /tmp/cookies.txt -b /tmp/cookies.txt https://$ip_addr/admin/ | grep 'Panel Hosting Management') ]]; then - token=$(grep csrftoken /tmp/cookies.txt | awk {'print $7'}) - if [[ $(curl -s -L -k -c /tmp/cookies.txt -b /tmp/cookies.txt \ - -d "username=$user&password=orchestra&csrfmiddlewaretoken=$token" \ - -e https://$ip_addr/admin/ \ - https://$ip_addr/admin/login/?next=/admin/ | grep 'Panel Hosting Management') ]]; then - echo "${bold} ** Orchestra appears to be working!${normal}" - else - echo "${bold} ** Err. Couldn't login :(${normal}" >&2 - fi - else - echo "${bold} ** Err. Orchestra is not responding responding on https://${ip_addr}/admin/${normal}" >&2 - fi - echo -} - - -function install_orchestra () { - dev=$1 - home=$2 - repo=$3 - - if [[ $dev ]]; then - # Install from source - python_path=$(python3 -c "import sys; print([path for path in sys.path if path.startswith('/usr/local/lib/python')][1]);") - if [[ -d $python_path/orchestra ]]; then - run sudo rm -fr $python_path/orchestra - fi - orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null || echo '') - if [[ ! $orch_version ]]; then - # First Orchestra installation - run sudo mkdir -p /usr/share/man/man1 - run sudo mkdir -p /usr/share/man/man7 - run sudo apt-get update - run sudo apt-get -y install git python3-pip - surun "git clone $repo $home/django-orchestra" || { - # Finishing partial installation - surun "export GIT_DIR=$home/django-orchestra/.git; git pull" - } - echo $home/django-orchestra/ | sudo tee "$python_path/orchestra.pth" - else - echo "You may want to execute 'git pull origin master'?" - fi - if [[ -L /usr/local/bin/orchestra-admin || -f /usr/local/bin/orchestra-admin ]]; then - run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} - fi - run sudo pip3 install -e $home/django-orchestra - run sudo orchestra-admin install_requirements --testing - else - # Install from pip - run sudo orchestra-admin install_requirements - run sudo pip3 install -e git+https://github.com/ribaguifi/django-orchestra.git#egg=django-orchestra - fi -} - - -function setup_database () { - dev=$1 - noinput=$2 - run sudo apt-get install -y postgresql - run sudo pip install psycopg2 - # Setup Database - if [[ $dev ]]; then - # Speeding up tests, don't do this in production! - . /usr/share/postgresql-common/init.d-functions - pg_version=$(psql --version | head -n1 | awk '{print $3}' | sed -e "s,\..*,,") - sudo sed -i \ - -e "s/^#fsync =\s*.*/fsync = off/" \ - -e "s/^#full_page_writes =\s*.*/full_page_writes = off/" \ - /etc/postgresql/${pg_version}/main/postgresql.conf - fi - run sudo service postgresql start - if [[ $noinput ]]; then - db_password=$(ps aux | sha256sum | base64 | head -c 10) - run sudo python3 -W ignore manage.py setuppostgres --noinput --db_password $db_password - else - run sudo python3 -W ignore manage.py setuppostgres - fi - if [[ $dev ]]; then - # Create database permissions are needed for running tests - sudo su postgres -c 'psql -c "ALTER USER orchestra CREATEDB;"' - fi - surun "python3 -W ignore manage.py migrate $noinput" -} - - -function create_orchestra_superuser () { - user=$1 - email=$2 - password=$3 - cat <<- EOF | surun "python3 -W ignore manage.py shell" - from orchestra.contrib.accounts.models import Account - if not Account.objects.filter(username="$user").exists(): - print('Creating orchestra superuser') - Account.objects.create_superuser("$user", "$email", "$password") - - EOF -} - - -print_help () { - cat <<- EOF - - ${bold}NAME${normal} - ${bold}deploy.sh${normal} - Deploy a django-orchestra project - - ${bold}SYNOPSIS${normal} - ${bold}deploy.sh${normal} [--noinput=USERNAME] [--dev] [--repo=GITREPO] [--projectname=NAME] - - ${bold}OPTIONS${normal} - ${bold}-n, --noinput=USERNAME${normal} - Execute the script without any user input, an existing system USERNAME is required. - requires the script to be executed as root user - - ${bold}-d, --dev${normal} - Perform a deployment suitable for development: - 1. debug mode - 2. dependencies for running tests - 3. access to source code - - ${bold}-r, --repo=GITREPO${normal} - Chose which repo use for development deployment - this option requires --dev option to be selected - https://github.com/glic3rinu/django-orchestra.git is used by default - - ${bold}-p, --projectname=NAME${normal} - Specify a project name, this will be asked on interactive mode - and name 'panel' will be used otherwise. - - ${bold}-h, --help${normal} - Display this message - - ${bold}EXAMPLES${normal} - deploy.sh - - deploy.sh --dev - - deploy.sh --dev --noinput orchestra - - EOF -} - - -function main () { - # Input validation - opts=$(getopt -o n:dr:h -l noinput:,dev,repo:,help -- "$@") || exit 1 - set -- $opts - - dev= - noinput= - user=$(whoami) - repo='https://github.com/ribaguifi/django-orchestra.git' - brepo= - project_name="panel" - bproject_name= - - while [ $# -gt 0 ]; do - case $1 in - -n|--noinput) user="${2:1:${#2}-2}"; noinput='--noinput'; shift ;; - -r|--repo) repo="${2:1:${#2}-2}"; brepo=true; shift ;; - -d|--dev) dev=true; ;; - -p|--project_name) project_name="${2:1:${#2}-2}"; bproject_name=true; shift ;; - -h|--help) print_help; exit 0 ;; - (--) shift; break;; - (-*) echo "$0: Err. - unrecognized option $1" 1>&2; exit 1;; - (*) break;; - esac - shift - done - unset OPTIND - unset opt - - if [[ ! $noinput ]]; then - if [[ $(whoami) == 'root' ]]; then - echo -e "\nErr. Interactive script should run as a regular user\n" >&2 - exit 2 - fi - run () { echo " ${bold}\$ ${@}${normal}"; ${@}; } - surun () { echo " ${bold}\$ ${@}${normal}"; ${@}; } - else - if [[ $(whoami) != 'root' ]]; then - echo -e "\nErr. --noinput should run as root\n" >&2 - exit 3 - elif ! id $user &> /dev/null; then - echo -e "\nErr. User $2 does not exist\n" >&2 - exit 4 - fi - run () { echo " ${bold}\$ ${@}${normal}"; ${@}; } - surun () { echo " ${bold}\$ su $user -c \"${@}\"${normal}"; su $user -c "${@}"; } - fi - if [[ ! $dev && $brepo ]]; then - echo -e "\nErr. --repo only makes sense with --dev\n" >&2 - exit 5 - fi - - sudo true - if [[ ! $noinput && ! $bproject_name ]]; then - while true; do - read -p "Enter a project name [panel]: " project_name - if [[ ! "$project_name" ]]; then - project_name="panel" - break - elif [[ ! $(echo "$project_name" | grep '^[_a-zA-Z]\w*$') ]]; then - if [[ ! $(echo "$project_name" | grep '^[_a-zA-Z]') ]]; then - message='make sure the name begins with a letter or underscore' - else - message='use only numbers, letters and underscores' - fi - echo "'$project_name' is not a valid %s name. Please $message." - else - break - fi - done - fi - - task=cronbeat - if [[ ! $noinput ]]; then - while true; do - read -p "Do you want to use celery or cronbeat (orchestra.contrib.tasks) for task execution [cronbeat]? " task - case $task in - 'celery' ) task=celery; break;; - 'cronbeat' ) task=cronbeat; break;; - '' ) task=cronbeat; break;; - * ) echo "Please answer celery or cronbeat.";; - esac - done - fi - - home=$(eval echo ~$user) - cd $home - - install_orchestra "$dev" $home $repo - if [[ ! -e $project_name ]]; then - surun "orchestra-admin startproject $project_name" - else - echo "Not deploying, $project_name already exists." - fi - cd $project_name - setup_database "$dev" "$noinput" - - if [[ $noinput ]]; then - create_orchestra_superuser $user $user@localhost orchestra - fi - - if [[ "$task" == "celery" ]]; then - run sudo apt-get install rabbitmq-server - run sudo python3 -W ignore manage.py setupcelery --username $user - else - surun "python3 -W ignore manage.py setupcronbeat" - surun "python3 -W ignore manage.py syncperiodictasks" - fi - - run sudo python3 -W ignore manage.py setuplog --noinput - - surun "python3 -W ignore manage.py collectstatic --noinput" - run sudo apt-get install nginx-full uwsgi uwsgi-plugin-python3 - run sudo python3 -W ignore manage.py setupnginx --user $user $noinput - run sudo python3 -W ignore manage.py restartservices - run sudo python3 -W ignore manage.py startservices - surun "python3 -W ignore manage.py check --deploy" - - - ip_addr=$(ip addr show eth0 | grep 'inet ' | sed -r "s/.*inet ([^\s]*).*/\1/" | cut -d'/' -f1) - if [[ ! $ip_addr ]]; then - ip_addr=127.0.0.1 - fi - - # Configure settings file into debug mode - if [[ $dev ]]; then - sed -i \ - -e "s/^\s*#\s*'debug_toolbar',/ 'debug_toolbar',/" \ - -e "s/^\s*#\s*'django_nose',/ 'django_nose',/" $project_name/settings.py - if [[ ! $(grep '^INTERNAL_IPS\s*=' $project_name/settings.py) ]]; then - echo "INTERNAL_IPS = ('$ip_addr',)" >> $project_name/settings.py - fi - fi - - test_orchestra $user $ip_addr -} - -# Wrap it all on a function to avoid partial executions when running through wget/curl -main $@ From cedb8d690b8de35af85af5e8359c1e07eac58e97 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Mon, 1 Feb 2021 12:37:45 +0100 Subject: [PATCH 22/22] swap tabs for spaces --- scripts/containers/deploy.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/containers/deploy.sh b/scripts/containers/deploy.sh index 45fdde20..4b7c3fc9 100644 --- a/scripts/containers/deploy.sh +++ b/scripts/containers/deploy.sh @@ -50,12 +50,12 @@ function install_orchestra () { if [[ -d $python_path/orchestra ]]; then run sudo rm -fr $python_path/orchestra fi - orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null || echo '') + orch_version=$(python3 -c "from orchestra import get_version; print(get_version());" 2> /dev/null || echo '') if [[ ! $orch_version ]]; then # First Orchestra installation - run sudo mkdir -p /usr/share/man/man1 - run sudo mkdir -p /usr/share/man/man7 - run sudo apt-get update + run sudo mkdir -p /usr/share/man/man1 + run sudo mkdir -p /usr/share/man/man7 + run sudo apt-get update run sudo apt-get -y install git python3-pip surun "git clone $repo $home/django-orchestra" || { # Finishing partial installation @@ -68,7 +68,7 @@ function install_orchestra () { if [[ -L /usr/local/bin/orchestra-admin || -f /usr/local/bin/orchestra-admin ]]; then run sudo rm -f /usr/local/bin/{orchestra-admin,orchestra-beat} fi - run sudo pip3 install -e $home/django-orchestra + run sudo pip3 install -e $home/django-orchestra run sudo orchestra-admin install_requirements --testing else # Install from pip @@ -87,7 +87,7 @@ function setup_database () { if [[ $dev ]]; then # Speeding up tests, don't do this in production! . /usr/share/postgresql-common/init.d-functions - pg_version=$(psql --version | head -n1 | awk '{print $3}' | sed -e "s,\..*,,") + pg_version=$(psql --version | head -n1 | awk '{print $3}' | sed -e "s,\..*,,") sudo sed -i \ -e "s/^#fsync =\s*.*/fsync = off/" \ -e "s/^#full_page_writes =\s*.*/full_page_writes = off/" \