Merge pull request #377 from DanielGrams/issue/376

Deployment with docker compose #376

.github/workflows/docker-build-push.yml

@@ -2,6 +2,8 @@ name: Docker build and push
 
 on:
   push:
+    branches:
+      - 'main'
     tags:
       - 'v*'
   workflow_dispatch:

Dockerfile

@@ -1,8 +1,8 @@
-FROM python:3.7-slim-buster
+FROM python:3.7
 
 # Add rsync
 RUN apt update -qq && apt upgrade -y && apt autoremove -y
-RUN apt install -y rsync && apt autoremove -y
+RUN apt install -y rsync curl && apt autoremove -y
 
 EXPOSE 5000
 
@@ -24,6 +24,7 @@ ENV STATIC_FILES_MIRROR=""
 
 # Install pip requirements
 COPY requirements.txt .
+RUN python -m pip install --upgrade pip
 RUN python -m pip install -r requirements.txt
 
 WORKDIR /app

deployment/docker-compose/.env.example

@@ -0,0 +1,26 @@
+POSTGRES_DATA_PATH=./tmp/data/postgres/data
+POSTGRES_BACKUP_PATH=./tmp/data/postgres/backups
+CACHE_PATH=./tmp/cache
+STATIC_PATH=./tmp/static
+FLUENTD_LOG_PATH=./tmp/logs/fluentd
+FLUENTD_CUSTOM_CONFIG_PATH=./tmp/config
+FLUENTD_DOCKER_CONTAINERS_PATH=/var/lib/docker/containers
+
+POSTGRES_USER=oveda
+POSTGRES_PASSWORD=
+POSTGRES_DB=oveda
+
+WEB_TAG=latest
+SERVER_NAME=
+PREFERRED_URL_SCHEME=https
+SECRET_KEY=
+SECURITY_PASSWORD_HASH=
+MAIL_SERVER=
+MAIL_PORT=
+MAIL_USERNAME=
+MAIL_PASSWORD=
+MAIL_DEFAULT_SENDER=
+MAIL_USE_TLS=True
+GOOGLE_MAPS_API_KEY=AIzaDummy
+JWT_PRIVATE_KEY=""
+JWT_PUBLIC_JWKS=''

deployment/docker-compose/README.md

@@ -0,0 +1,31 @@
+# Deployment with Docker compose
+
+## Configure
+
+Copy example.env to .env and enter values.
+
+## Initialize
+
+```sh
+./init.sh
+```
+
+## Start
+
+```sh
+docker compose up --force-recreate --detach
+```
+
+## Update app
+
+Adjust `WEB_TAG` in .env if necessary.
+
+```sh
+./update.sh
+```
+
+## Execute commands in web container
+
+```sh
+docker compose exec -it web /bin/sh
+```

deployment/docker-compose/docker-compose.yml

@@ -0,0 +1,82 @@
+version: "3.9"
+name: "oveda"
+
+services:
+  db:
+    image: postgis/postgis:12-3.1
+    restart: always
+    healthcheck:
+      test: "pg_isready --username=${POSTGRES_USER} && psql --username=${POSTGRES_USER} --list"
+      start_period: "5s"
+    ports:
+      - 5434:5432
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    volumes:
+      - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data
+
+  db-backup:
+    image: prodrigestivill/postgres-backup-local:12
+    restart: always
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_HOST: db
+      POSTGRES_EXTRA_OPTS: "-Z6 -c"
+      SCHEDULE: "0 0 22 * * *"
+    volumes:
+      - ${POSTGRES_BACKUP_PATH}:/backups
+    depends_on:
+      db:
+        condition: service_healthy
+
+  web:
+    image: danielgrams/gsevpt:${WEB_TAG}
+    restart: always
+    healthcheck:
+      test: "curl -f ${SERVER_NAME}/up"
+      interval: "60s"
+      timeout: "5s"
+      start_period: "5s"
+    ports:
+      - "5000:5000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      FLASK_APP: main.py
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB}
+      SECRET_KEY: ${SECRET_KEY}
+      SECURITY_PASSWORD_HASH: ${SECURITY_PASSWORD_HASH}
+      MAIL_DEFAULT_SENDER: ${MAIL_DEFAULT_SENDER}
+      MAIL_PASSWORD: ${MAIL_PASSWORD}
+      MAIL_PORT: ${MAIL_PORT}
+      MAIL_SERVER: ${MAIL_SERVER}
+      MAIL_USE_TLS: ${MAIL_USE_TLS}
+      MAIL_USERNAME: ${MAIL_USERNAME}
+      GOOGLE_MAPS_API_KEY: ${GOOGLE_MAPS_API_KEY}
+      SERVER_NAME: ${SERVER_NAME}
+      PREFERRED_URL_SCHEME: ${PREFERRED_URL_SCHEME}
+      GUNICORN_ACCESS_LOG: "-"
+      STATIC_FILES_MIRROR: /static
+      CACHE_PATH: tmp
+      JWT_PRIVATE_KEY: ${JWT_PRIVATE_KEY}
+      JWT_PUBLIC_JWKS: ${JWT_PUBLIC_JWKS}
+    volumes:
+      - ${CACHE_PATH}:/app/project/tmp
+      - ${STATIC_PATH}:/static
+    depends_on:
+      db:
+        condition: service_healthy
+
+  fluentd:
+    image: danielgrams/fluentd
+    restart: always
+    environment:
+      FLUENTD_CONF: fluentd-custom.config
+    volumes:
+      - ${FLUENTD_LOG_PATH}:/fluentd/log
+      - ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config:/fluentd/etc/fluentd-custom.config
+      - ${FLUENTD_DOCKER_CONTAINERS_PATH}:/fluentd/containers

deployment/docker-compose/fluentd-custom.config

@@ -0,0 +1,60 @@
+# Tail docker logs
+<source>
+  @type tail
+  read_from_head true
+  path /fluentd/containers/*/*-json.log
+  pos_file /fluentd/log/docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.*
+  format json
+</source>
+
+# Add container id
+<filter docker.fluentd.containers.*.*.log>
+  @type record_transformer
+  <record>
+    container_id ${tag_parts[3]}
+  </record>
+</filter>
+
+# Errors only
+<filter docker.**>
+  @type grep
+  <regexp>
+    key log
+    pattern /\[error\]|\| error \|/i
+  </regexp>
+</filter>
+
+<match docker.**>
+  @type copy
+
+  # Write errors to file
+  <store>
+    @type file
+    path /fluentd/log/docker-error
+  </store>
+
+  # Tag daily errors
+  <store>
+    @type grepcounter
+    count_interval 14400 # = 4 hours
+    input_key log
+    threshold 1
+    add_tag_prefix daily_error
+  </store>
+</match>
+
+# Send daily error mail
+#<match daily_error.docker.**>
+#  @type mail
+#  host [Host]
+#  port [Port]
+#  user [User/Email]
+#  password [Password]
+#  from [User/Email]
+#  to [Recipient]
+#  subject 'Error alert fluentd'
+#  message Error occured %s times
+#  message_out_keys count
+#</match>

deployment/docker-compose/init.sh

@@ -0,0 +1,10 @@
+set -e
+source .env
+
+mkdir -p ${POSTGRES_DATA_PATH}
+mkdir -p ${POSTGRES_BACKUP_PATH}
+mkdir -p ${CACHE_PATH}
+mkdir -p ${STATIC_PATH}
+mkdir -p ${FLUENTD_LOG_PATH}
+mkdir -p ${FLUENTD_CUSTOM_CONFIG_PATH}
+cp ./fluentd-custom.config ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config

deployment/docker-compose/nginx.config.example

@@ -0,0 +1,42 @@
+location ^~ /image/ {
+	root "/var/www/vhosts/oveda.de/cache/img";
+	expires 1h;
+
+	location ~ ^/image/(?<id>[0-9]+)/(?<hash>[0-9]+) {
+		if ($arg_s = '') {
+			rewrite (.*) $1?s=500 last;
+		}
+		try_files /${id}-${hash}-${arg_s}-${arg_s}.png /${id}-${hash}-${arg_s}-${arg_s}.jpg @docker;
+	}
+
+	location ~ ^/image/(?<id>[0-9]+) {
+		if ($arg_s = '') {
+			rewrite (.*) $1?s=500 last;
+		}
+		try_files /${id}-${arg_s}-${arg_s}.png /${id}-${arg_s}-${arg_s}.jpg @docker;
+	}
+}
+location ^~ /static/ {
+	alias "/var/www/vhosts/oveda.de/static/";
+	expires 1h;
+}
+location ^~ /dump/ {
+	alias "/var/www/vhosts/oveda.de/cache/dump/";
+}
+location ^~ /sitemap.xml {
+	alias "/var/www/vhosts/oveda.de/cache/sitemap.xml";
+}
+location ^~ /robots.txt {
+	alias "/var/www/vhosts/oveda.de/cache/robots.txt";
+}
+location ^~ /favicon.ico {
+	alias "/var/www/vhosts/oveda.de/static/favicon.ico";
+	expires 12h;
+}
+location @docker {
+	proxy_pass http://0.0.0.0:5000;
+	proxy_set_header Host              $host;
+	proxy_set_header X-Real-IP         $remote_addr;
+	proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+}

deployment/docker-compose/update.sh

@@ -0,0 +1,6 @@
+set -e
+
+docker compose pull web
+docker compose stop web
+docker compose exec db-backup /backup.sh
+docker compose up --detach --force-recreate web

doc/deployment.md

@@ -44,6 +44,8 @@ Jobs that should run on a regular basis.
 ```sh
 flask event update-recurring-dates
 flask dump all
+flask seo generate-sitemap --pinggoogle
+flask seo generate-robots-txt
 ```
 
 ## Administration

doc/development.md

@@ -68,7 +68,7 @@ pybabel compile -d project/translations
 ### Build image
 
 ```sh
-docker build -t gsevpt:latest .
+docker build -t danielgrams/gsevpt:latest .
 ```
 
 ### Run container with existing postgres server

migrations/versions/091deace5f08_.py

@@ -18,7 +18,6 @@ depends_on = None
 
 
 def upgrade():
-
     bind = op.get_bind()
     bind.execute(text("create extension if not exists postgis;"))
 

project/__init__.py

@@ -14,6 +14,11 @@ from flask_wtf.csrf import CSRFProtect
 
 from project.custom_session_interface import CustomSessionInterface
 
+
+def getenv_bool(name: str, default: str = "False"):  # pragma: no cover
+    return os.getenv(name, default).lower() in ("true", "1", "t")
+
+
 # Create app
 app = Flask(__name__)
 app.config["SQLALCHEMY_DATABASE_URI"] = os.environ["DATABASE_URL"]
@@ -95,15 +100,15 @@ app.config["WTF_CSRF_CHECK_DEFAULT"] = False
 # Mail
 mail_server = os.getenv("MAIL_SERVER")
 
-if mail_server is None:
+if not mail_server:
     app.config["MAIL_SUPPRESS_SEND"] = True
     app.config["MAIL_DEFAULT_SENDER"] = "test@oveda.de"
 else:  # pragma: no cover
     app.config["MAIL_SUPPRESS_SEND"] = False
     app.config["MAIL_SERVER"] = mail_server
     app.config["MAIL_PORT"] = os.getenv("MAIL_PORT")
-    app.config["MAIL_USE_TLS"] = os.getenv("MAIL_USE_TLS", True)
-    app.config["MAIL_USE_SSL"] = os.getenv("MAIL_USE_SSL", False)
+    app.config["MAIL_USE_TLS"] = getenv_bool("MAIL_USE_TLS", "True")
+    app.config["MAIL_USE_SSL"] = getenv_bool("MAIL_USE_SSL", "False")
     app.config["MAIL_USERNAME"] = os.getenv("MAIL_USERNAME")
     app.config["MAIL_PASSWORD"] = os.getenv("MAIL_PASSWORD")
     app.config["MAIL_DEFAULT_SENDER"] = os.getenv("MAIL_DEFAULT_SENDER")

project/services/importer/ld_json_importer.py

@@ -343,7 +343,6 @@ class LdJsonImporter:
         image_items = image if isinstance(image, list) else [image]
 
         for image_item in image_items:
-
             if isinstance(image_item, str):
                 image_item = {"url": image_item}
 

project/views/event.py

@@ -398,7 +398,6 @@ def send_referenced_event_changed_mails(event):
     # Alle Referenzen
     references = EventReference.query.filter(EventReference.event_id == event.id).all()
     for reference in references:
-
         # Alle Mitglieder der AdminUnit, die das Recht haben, Requests zu verifizieren
         members = get_admin_unit_members_with_permission(
             reference.admin_unit_id, "reference_request:verify"

project/views/root.py

@@ -5,7 +5,7 @@ from flask import redirect, render_template, request, send_from_directory, url_f
 from flask_babelex import gettext
 from markupsafe import Markup
 
-from project import app, cache_path, dump_path, robots_txt_file, sitemap_file
+from project import app, cache_path, db, dump_path, robots_txt_file, sitemap_file
 from project.services.admin import upsert_settings
 from project.views.utils import track_analytics
 
@@ -32,6 +32,12 @@ def home():
     )
 
 
+@app.route("/up")
+def up():
+    db.engine.execute("SELECT 1")
+    return "OK"
+
+
 @app.route("/tos")
 def tos():
     title = gettext("Terms of service")

requirements.txt

@@ -10,13 +10,13 @@ Authlib==0.15.3
 Babel==2.9.1
 bcrypt==3.2.0
 beautifulsoup4==4.9.3
-black==20.8b1
+black==23.1.0
 blinker==1.4
 certifi==2020.12.5
 cffi==1.14.4
 cfgv==3.2.0
 chardet==3.0.4
-click==7.1.2
+click==8.1.3
 colour==0.1.5
 coverage==5.5
 coveralls==2.2.0
@@ -68,11 +68,12 @@ mistune==0.8.4
 mypy-extensions==0.4.3
 nodeenv==1.5.0
 oauthlib==3.1.0
-packaging==20.8
+packaging==23.0
 passlib==1.7.4
-pathspec==0.8.1
+pathspec==0.11.0
 pilkit==2.0
 Pillow==9.0.0
+platformdirs==3.1.0
 pluggy==0.13.1
 pre-commit==2.9.3
 psycopg2-binary==2.8.6
@@ -106,8 +107,9 @@ SQLAlchemy-Utils==0.36.8
 swagger-spec-validator==2.7.3
 TatSu==4.4.0
 toml==0.10.2
-typed-ast==1.4.1
-typing-extensions==3.7.4.3
+tomli==2.0.1
+typed-ast==1.5.4
+typing_extensions==4.5.0
 urllib3==1.26.5
 URLObject==2.4.3
 validators==0.18.2

tests/services/importer/test_event_importer.py

@@ -11,7 +11,6 @@ def test_import(client, seeder, utils, app, shared_datadir, requests_mock):
     params = (utils, admin_unit_id, shared_datadir)
 
     with app.app_context():
-
         _assert_import_event(
             params,
             "facebook.html",

tests/views/test_root.py

@@ -12,6 +12,10 @@ def test_home(client, seeder, utils):
     utils.assert_response_redirect(response, "home")
 
 
+def test_up(app, utils):
+    utils.get_ok("up")
+
+
 def test_organizations(client, seeder, utils):
     url = utils.get_url("organizations")
     utils.get_ok(url)