diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 4fc2db8..746ae75 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -2,6 +2,8 @@ name: Docker build and push on: push: + branches: + - 'main' tags: - 'v*' workflow_dispatch: diff --git a/Dockerfile b/Dockerfile index e7d2754..48d8ad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.7-slim-buster +FROM python:3.7 # Add rsync RUN apt update -qq && apt upgrade -y && apt autoremove -y -RUN apt install -y rsync && apt autoremove -y +RUN apt install -y rsync curl && apt autoremove -y EXPOSE 5000 @@ -24,6 +24,7 @@ ENV STATIC_FILES_MIRROR="" # Install pip requirements COPY requirements.txt . +RUN python -m pip install --upgrade pip RUN python -m pip install -r requirements.txt WORKDIR /app diff --git a/deployment/docker-compose/.env.example b/deployment/docker-compose/.env.example new file mode 100644 index 0000000..bd16dbe --- /dev/null +++ b/deployment/docker-compose/.env.example @@ -0,0 +1,26 @@ +POSTGRES_DATA_PATH=./tmp/data/postgres/data +POSTGRES_BACKUP_PATH=./tmp/data/postgres/backups +CACHE_PATH=./tmp/cache +STATIC_PATH=./tmp/static +FLUENTD_LOG_PATH=./tmp/logs/fluentd +FLUENTD_CUSTOM_CONFIG_PATH=./tmp/config +FLUENTD_DOCKER_CONTAINERS_PATH=/var/lib/docker/containers + +POSTGRES_USER=oveda +POSTGRES_PASSWORD= +POSTGRES_DB=oveda + +WEB_TAG=latest +SERVER_NAME= +PREFERRED_URL_SCHEME=https +SECRET_KEY= +SECURITY_PASSWORD_HASH= +MAIL_SERVER= +MAIL_PORT= +MAIL_USERNAME= +MAIL_PASSWORD= +MAIL_DEFAULT_SENDER= +MAIL_USE_TLS=True +GOOGLE_MAPS_API_KEY=AIzaDummy +JWT_PRIVATE_KEY="" +JWT_PUBLIC_JWKS='' \ No newline at end of file diff --git a/deployment/docker-compose/README.md b/deployment/docker-compose/README.md new file mode 100644 index 0000000..5e4797c --- /dev/null +++ b/deployment/docker-compose/README.md @@ -0,0 +1,31 @@ +# Deployment with Docker compose + +## Configure + +Copy example.env to .env and enter values. + +## Initialize + +```sh +./init.sh +``` + +## Start + +```sh +docker compose up --force-recreate --detach +``` + +## Update app + +Adjust `WEB_TAG` in .env if necessary. + +```sh +./update.sh +``` + +## Execute commands in web container + +```sh +docker compose exec -it web /bin/sh +``` diff --git a/deployment/docker-compose/docker-compose.yml b/deployment/docker-compose/docker-compose.yml new file mode 100644 index 0000000..9dba2e9 --- /dev/null +++ b/deployment/docker-compose/docker-compose.yml @@ -0,0 +1,82 @@ +version: "3.9" +name: "oveda" + +services: + db: + image: postgis/postgis:12-3.1 + restart: always + healthcheck: + test: "pg_isready --username=${POSTGRES_USER} && psql --username=${POSTGRES_USER} --list" + start_period: "5s" + ports: + - 5434:5432 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - ${POSTGRES_DATA_PATH}:/var/lib/postgresql/data + + db-backup: + image: prodrigestivill/postgres-backup-local:12 + restart: always + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: db + POSTGRES_EXTRA_OPTS: "-Z6 -c" + SCHEDULE: "0 0 22 * * *" + volumes: + - ${POSTGRES_BACKUP_PATH}:/backups + depends_on: + db: + condition: service_healthy + + web: + image: danielgrams/gsevpt:${WEB_TAG} + restart: always + healthcheck: + test: "curl -f ${SERVER_NAME}/up" + interval: "60s" + timeout: "5s" + start_period: "5s" + ports: + - "5000:5000" + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + FLASK_APP: main.py + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB} + SECRET_KEY: ${SECRET_KEY} + SECURITY_PASSWORD_HASH: ${SECURITY_PASSWORD_HASH} + MAIL_DEFAULT_SENDER: ${MAIL_DEFAULT_SENDER} + MAIL_PASSWORD: ${MAIL_PASSWORD} + MAIL_PORT: ${MAIL_PORT} + MAIL_SERVER: ${MAIL_SERVER} + MAIL_USE_TLS: ${MAIL_USE_TLS} + MAIL_USERNAME: ${MAIL_USERNAME} + GOOGLE_MAPS_API_KEY: ${GOOGLE_MAPS_API_KEY} + SERVER_NAME: ${SERVER_NAME} + PREFERRED_URL_SCHEME: ${PREFERRED_URL_SCHEME} + GUNICORN_ACCESS_LOG: "-" + STATIC_FILES_MIRROR: /static + CACHE_PATH: tmp + JWT_PRIVATE_KEY: ${JWT_PRIVATE_KEY} + JWT_PUBLIC_JWKS: ${JWT_PUBLIC_JWKS} + volumes: + - ${CACHE_PATH}:/app/project/tmp + - ${STATIC_PATH}:/static + depends_on: + db: + condition: service_healthy + + fluentd: + image: danielgrams/fluentd + restart: always + environment: + FLUENTD_CONF: fluentd-custom.config + volumes: + - ${FLUENTD_LOG_PATH}:/fluentd/log + - ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config:/fluentd/etc/fluentd-custom.config + - ${FLUENTD_DOCKER_CONTAINERS_PATH}:/fluentd/containers diff --git a/deployment/docker-compose/fluentd-custom.config b/deployment/docker-compose/fluentd-custom.config new file mode 100644 index 0000000..d09bf34 --- /dev/null +++ b/deployment/docker-compose/fluentd-custom.config @@ -0,0 +1,60 @@ +# Tail docker logs + + @type tail + read_from_head true + path /fluentd/containers/*/*-json.log + pos_file /fluentd/log/docker.pos + time_format %Y-%m-%dT%H:%M:%S + tag docker.* + format json + + +# Add container id + + @type record_transformer + + container_id ${tag_parts[3]} + + + +# Errors only + + @type grep + + key log + pattern /\[error\]|\| error \|/i + + + + + @type copy + + # Write errors to file + + @type file + path /fluentd/log/docker-error + + + # Tag daily errors + + @type grepcounter + count_interval 14400 # = 4 hours + input_key log + threshold 1 + add_tag_prefix daily_error + + + +# Send daily error mail +# +# @type mail +# host [Host] +# port [Port] +# user [User/Email] +# password [Password] +# from [User/Email] +# to [Recipient] +# subject 'Error alert fluentd' +# message Error occured %s times +# message_out_keys count +# \ No newline at end of file diff --git a/deployment/docker-compose/init.sh b/deployment/docker-compose/init.sh new file mode 100755 index 0000000..71a3ee4 --- /dev/null +++ b/deployment/docker-compose/init.sh @@ -0,0 +1,10 @@ +set -e +source .env + +mkdir -p ${POSTGRES_DATA_PATH} +mkdir -p ${POSTGRES_BACKUP_PATH} +mkdir -p ${CACHE_PATH} +mkdir -p ${STATIC_PATH} +mkdir -p ${FLUENTD_LOG_PATH} +mkdir -p ${FLUENTD_CUSTOM_CONFIG_PATH} +cp ./fluentd-custom.config ${FLUENTD_CUSTOM_CONFIG_PATH}/fluentd-custom.config \ No newline at end of file diff --git a/deployment/docker-compose/nginx.config.example b/deployment/docker-compose/nginx.config.example new file mode 100644 index 0000000..fe99696 --- /dev/null +++ b/deployment/docker-compose/nginx.config.example @@ -0,0 +1,42 @@ +location ^~ /image/ { + root "/var/www/vhosts/oveda.de/cache/img"; + expires 1h; + + location ~ ^/image/(?[0-9]+)/(?[0-9]+) { + if ($arg_s = '') { + rewrite (.*) $1?s=500 last; + } + try_files /${id}-${hash}-${arg_s}-${arg_s}.png /${id}-${hash}-${arg_s}-${arg_s}.jpg @docker; + } + + location ~ ^/image/(?[0-9]+) { + if ($arg_s = '') { + rewrite (.*) $1?s=500 last; + } + try_files /${id}-${arg_s}-${arg_s}.png /${id}-${arg_s}-${arg_s}.jpg @docker; + } +} +location ^~ /static/ { + alias "/var/www/vhosts/oveda.de/static/"; + expires 1h; +} +location ^~ /dump/ { + alias "/var/www/vhosts/oveda.de/cache/dump/"; +} +location ^~ /sitemap.xml { + alias "/var/www/vhosts/oveda.de/cache/sitemap.xml"; +} +location ^~ /robots.txt { + alias "/var/www/vhosts/oveda.de/cache/robots.txt"; +} +location ^~ /favicon.ico { + alias "/var/www/vhosts/oveda.de/static/favicon.ico"; + expires 12h; +} +location @docker { + proxy_pass http://0.0.0.0:5000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} \ No newline at end of file diff --git a/deployment/docker-compose/update.sh b/deployment/docker-compose/update.sh new file mode 100755 index 0000000..09075b4 --- /dev/null +++ b/deployment/docker-compose/update.sh @@ -0,0 +1,6 @@ +set -e + +docker compose pull web +docker compose stop web +docker compose exec db-backup /backup.sh +docker compose up --detach --force-recreate web \ No newline at end of file diff --git a/doc/deployment.md b/doc/deployment.md index c93db6d..1ad297f 100644 --- a/doc/deployment.md +++ b/doc/deployment.md @@ -44,6 +44,8 @@ Jobs that should run on a regular basis. ```sh flask event update-recurring-dates flask dump all +flask seo generate-sitemap --pinggoogle +flask seo generate-robots-txt ``` ## Administration diff --git a/doc/development.md b/doc/development.md index 7e195dd..5a2eac4 100644 --- a/doc/development.md +++ b/doc/development.md @@ -68,7 +68,7 @@ pybabel compile -d project/translations ### Build image ```sh -docker build -t gsevpt:latest . +docker build -t danielgrams/gsevpt:latest . ``` ### Run container with existing postgres server diff --git a/migrations/versions/091deace5f08_.py b/migrations/versions/091deace5f08_.py index 7e08304..00907ae 100644 --- a/migrations/versions/091deace5f08_.py +++ b/migrations/versions/091deace5f08_.py @@ -18,7 +18,6 @@ depends_on = None def upgrade(): - bind = op.get_bind() bind.execute(text("create extension if not exists postgis;")) diff --git a/project/__init__.py b/project/__init__.py index 763986c..44135c1 100644 --- a/project/__init__.py +++ b/project/__init__.py @@ -14,6 +14,11 @@ from flask_wtf.csrf import CSRFProtect from project.custom_session_interface import CustomSessionInterface + +def getenv_bool(name: str, default: str = "False"): # pragma: no cover + return os.getenv(name, default).lower() in ("true", "1", "t") + + # Create app app = Flask(__name__) app.config["SQLALCHEMY_DATABASE_URI"] = os.environ["DATABASE_URL"] @@ -95,15 +100,15 @@ app.config["WTF_CSRF_CHECK_DEFAULT"] = False # Mail mail_server = os.getenv("MAIL_SERVER") -if mail_server is None: +if not mail_server: app.config["MAIL_SUPPRESS_SEND"] = True app.config["MAIL_DEFAULT_SENDER"] = "test@oveda.de" else: # pragma: no cover app.config["MAIL_SUPPRESS_SEND"] = False app.config["MAIL_SERVER"] = mail_server app.config["MAIL_PORT"] = os.getenv("MAIL_PORT") - app.config["MAIL_USE_TLS"] = os.getenv("MAIL_USE_TLS", True) - app.config["MAIL_USE_SSL"] = os.getenv("MAIL_USE_SSL", False) + app.config["MAIL_USE_TLS"] = getenv_bool("MAIL_USE_TLS", "True") + app.config["MAIL_USE_SSL"] = getenv_bool("MAIL_USE_SSL", "False") app.config["MAIL_USERNAME"] = os.getenv("MAIL_USERNAME") app.config["MAIL_PASSWORD"] = os.getenv("MAIL_PASSWORD") app.config["MAIL_DEFAULT_SENDER"] = os.getenv("MAIL_DEFAULT_SENDER") diff --git a/project/services/importer/ld_json_importer.py b/project/services/importer/ld_json_importer.py index 131d6b1..6437f67 100644 --- a/project/services/importer/ld_json_importer.py +++ b/project/services/importer/ld_json_importer.py @@ -343,7 +343,6 @@ class LdJsonImporter: image_items = image if isinstance(image, list) else [image] for image_item in image_items: - if isinstance(image_item, str): image_item = {"url": image_item} diff --git a/project/views/event.py b/project/views/event.py index 25af426..8126a11 100644 --- a/project/views/event.py +++ b/project/views/event.py @@ -398,7 +398,6 @@ def send_referenced_event_changed_mails(event): # Alle Referenzen references = EventReference.query.filter(EventReference.event_id == event.id).all() for reference in references: - # Alle Mitglieder der AdminUnit, die das Recht haben, Requests zu verifizieren members = get_admin_unit_members_with_permission( reference.admin_unit_id, "reference_request:verify" diff --git a/project/views/root.py b/project/views/root.py index 4dd80a1..5289e3a 100644 --- a/project/views/root.py +++ b/project/views/root.py @@ -5,7 +5,7 @@ from flask import redirect, render_template, request, send_from_directory, url_f from flask_babelex import gettext from markupsafe import Markup -from project import app, cache_path, dump_path, robots_txt_file, sitemap_file +from project import app, cache_path, db, dump_path, robots_txt_file, sitemap_file from project.services.admin import upsert_settings from project.views.utils import track_analytics @@ -32,6 +32,12 @@ def home(): ) +@app.route("/up") +def up(): + db.engine.execute("SELECT 1") + return "OK" + + @app.route("/tos") def tos(): title = gettext("Terms of service") diff --git a/requirements.txt b/requirements.txt index 4841bd6..f5ffbfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,13 +10,13 @@ Authlib==0.15.3 Babel==2.9.1 bcrypt==3.2.0 beautifulsoup4==4.9.3 -black==20.8b1 +black==23.1.0 blinker==1.4 certifi==2020.12.5 cffi==1.14.4 cfgv==3.2.0 chardet==3.0.4 -click==7.1.2 +click==8.1.3 colour==0.1.5 coverage==5.5 coveralls==2.2.0 @@ -68,11 +68,12 @@ mistune==0.8.4 mypy-extensions==0.4.3 nodeenv==1.5.0 oauthlib==3.1.0 -packaging==20.8 +packaging==23.0 passlib==1.7.4 -pathspec==0.8.1 +pathspec==0.11.0 pilkit==2.0 Pillow==9.0.0 +platformdirs==3.1.0 pluggy==0.13.1 pre-commit==2.9.3 psycopg2-binary==2.8.6 @@ -106,8 +107,9 @@ SQLAlchemy-Utils==0.36.8 swagger-spec-validator==2.7.3 TatSu==4.4.0 toml==0.10.2 -typed-ast==1.4.1 -typing-extensions==3.7.4.3 +tomli==2.0.1 +typed-ast==1.5.4 +typing_extensions==4.5.0 urllib3==1.26.5 URLObject==2.4.3 validators==0.18.2 diff --git a/tests/services/importer/test_event_importer.py b/tests/services/importer/test_event_importer.py index c53a703..7800292 100644 --- a/tests/services/importer/test_event_importer.py +++ b/tests/services/importer/test_event_importer.py @@ -11,7 +11,6 @@ def test_import(client, seeder, utils, app, shared_datadir, requests_mock): params = (utils, admin_unit_id, shared_datadir) with app.app_context(): - _assert_import_event( params, "facebook.html", diff --git a/tests/views/test_root.py b/tests/views/test_root.py index 778e786..cd3a72c 100644 --- a/tests/views/test_root.py +++ b/tests/views/test_root.py @@ -12,6 +12,10 @@ def test_home(client, seeder, utils): utils.assert_response_redirect(response, "home") +def test_up(app, utils): + utils.get_ok("up") + + def test_organizations(client, seeder, utils): url = utils.get_url("organizations") utils.get_ok(url)