Add performance tuning for nextcloud app

Fix nextcloud cron jobs never being run

playbooks/configure-mgmt.yml

@@ -24,9 +24,9 @@
 - name: Configure local accounts
   hosts: all
   vars_files:
-    - vars/accounts.yml
+    - vars/accounts.yaml
-    - vars/secrets/passwords.yml
+    - vars/secrets/passwords.yaml
-    - vars/sshkeys.yml
+    - vars/sshkeys.yaml
   tasks:
     - name: Create omni group
       become: true

playbooks/deploy-compose.yml

@@ -46,6 +46,16 @@
         group: "{{ omni_compose_apps[_runtime_application].account.name }}"
         mode: 0750
 
+    - name: Create datastore assets
+      become: true
+      template:
+        src: "{{ item.src }}"
+        dest: "{{ omni_datastore_mount }}{{ omni_compose_apps[_runtime_application].datastore }}/{{ item.name }}"
+        owner: "{{ omni_compose_apps[_runtime_application].account.name }}"
+        group: "{{ omni_compose_apps[_runtime_application].account.name }}"
+        mode: "{{ item.permissions | default(0644) }}"
+      loop: "{{ omni_compose_apps[_runtime_application].assets | default([]) }}"
+
 
 - name: Configure docker stack
   hosts: jupiter
@@ -70,6 +80,15 @@
         owner: "{{ ansible_user }}"
         group: docker
         mode: 0640
+      register: _stack_file_state
+
+    - name: Remove the existing stack
+      when: _stack_file_state.changed is true or omni_compose_apps[_runtime_application].force_clean | default(false) is true
+      docker_stack:
+        name: "{{ _runtime_application }}"
+        state: absent
+        compose:
+          - "{{ omni_docker_configs }}/{{ _runtime_application }}/docker-compose.yaml"
 
     - name: Deploy the stack
       docker_stack:

playbooks/provision-common.yml

@@ -5,7 +5,7 @@
 - name: Configure system settings
   hosts: all
   vars_files:
-    - vars/packages.yml
+    - vars/packages.yaml
   pre_tasks:
     - import_tasks: tasks/centos-8-kernelplus.yml
   tasks:

resources/docker-compose/nextcloud.yaml.j2

@@ -0,0 +1,144 @@
+---
+version: "{{ omni_compose_version | string }}"
+
+
+x-server-env: &server-env
+  NEXTCLOUD_DATA_DIR: /data/
+  NEXTCLOUD_ADMIN_USER: admin
+  NEXTCLOUD_ADMIN_PASSWORD: {{ omni_compose_app_secrets.nextcloud.admin_password }}
+  NEXTCLOUD_TRUSTED_DOMAINS: localhost {{ inventory_hostname }} {{ omni_compose_apps.nextcloud.published.host }}
+  MYSQL_DATABASE: nextcloud
+  MYSQL_USER: root
+  MYSQL_PASSWORD: {{ omni_compose_app_secrets.nextcloud.database_password }}
+  MYSQL_HOST: database
+  REDIS_HOST: cache
+  PHP_MEMORY_LIMIT: "12G"
+  PHP_UPLOAD_LIMIT: "6G"
+  PHP_INI_SCAN_DIR: /usr/local/etc/php/conf.d:/var/www/html/
+
+
+networks:
+  nextcloud:
+    name: nextcloud
+    driver: overlay
+    ipam:
+      driver: default
+      config:
+        - subnet: {{ omni_compose_apps.nextcloud.networks.main }}
+
+
+volumes:
+  database:
+    name: datastore{{ omni_compose_apps.nextcloud.datastore }}/database
+    driver: glusterfs
+  data:
+    name: datastore/{{ omni_compose_apps.nextcloud.datastore }}/userdata
+    driver: glusterfs
+  config:
+    name: datastore{{ omni_compose_apps.nextcloud.datastore }}/config
+    driver: glusterfs
+  proxy:
+    name: datastore{{ omni_compose_apps.nextcloud.datastore }}/proxy
+    driver: glusterfs
+
+
+services:
+  database:
+    image: mariadb:{{ omni_compose_apps.nextcloud.versions.database | default(omni_compose_apps.nextcloud.versions.default) }}
+    hostname: nextcloud-database
+    networks:
+      - nextcloud
+    volumes:
+      - type: volume
+        source: database
+        target: /var/lib/mysql
+        read_only: false
+      - type: volume
+        source: proxy
+        target: /etc/mysql/conf.d
+        read_only: true
+    environment:
+      MYSQL_ROOT_PASSWORD: {{ omni_compose_app_secrets.nextcloud.database_password }}
+      MYSQL_DATABASE: nextcloud
+    deploy:
+      replicas: 1
+
+  cache:
+    image: redis:{{ omni_compose_apps.nextcloud.versions.cache | default(omni_compose_apps.nextcloud.versions.default) }}
+    hostname: nextcloud-cache
+    networks:
+      - nextcloud
+    deploy:
+      replicas: 1
+
+  proxy:
+    image: nginx:{{ omni_compose_apps.nextcloud.versions.proxy | default(omni_compose_apps.nextcloud.versions.default) }}
+    hostname: nextcloud-proxy
+    networks:
+      - nextcloud
+    depends_on:
+      - server
+    ports:
+      - published: {{ omni_compose_apps.nextcloud.published.ports.80 }}
+        target: 80
+        protocol: tcp
+        mode: ingress
+    volumes:
+      - type: volume
+        source: config
+        target: /usr/share/nginx/nextcloud
+        read_only: true
+      - type: volume
+        source: proxy
+        target: /etc/nginx/conf.d
+        read_only: true
+    deploy:
+      replicas: 1
+
+  server:
+    image: nextcloud:{{ omni_compose_apps.nextcloud.versions.server | default(omni_compose_apps.nextcloud.versions.default) }}
+    hostname: nextcloud-server
+    user: "{{ omni_compose_apps.nextcloud.account.uid }}"
+    networks:
+      - nextcloud
+    depends_on:
+      - database
+      - cache
+    volumes:
+      - type: volume
+        source: data
+        target: /data
+        read_only: false
+      - type: volume
+        source: config
+        target: /var/www/html
+        read_only: false
+    environment: *server-env
+    deploy:
+      replicas: 1
+
+  cron:
+    image: nextcloud:{{ omni_compose_apps.nextcloud.versions.server | default(omni_compose_apps.nextcloud.versions.default) }}
+    command: php /var/www/html/cron.php
+    hostname: nextcloud-cron
+    user: "{{ omni_compose_apps.nextcloud.account.uid }}"
+    networks:
+      - nextcloud
+    depends_on:
+      - database
+      - cache
+    volumes:
+      - type: volume
+        source: data
+        target: /data
+        read_only: false
+      - type: volume
+        source: config
+        target: /var/www/html
+        read_only: false
+    environment: *server-env
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: any
+        delay: "4m"

resources/docker-compose/plex.yaml.j2

@@ -23,6 +23,9 @@ volumes:
       type: nfs
       o: "addr=plexistore.tre2.local,ro"
       device: ":/nfs/plex"
+  plex-personal:
+    name: datastore/media
+    driver: glusterfs
 
 
 services:
@@ -73,6 +76,10 @@ services:
         source: plex-data
         target: /data
         read_only: true
+      - type: volume
+        source: plex-personal
+        target: /personal
+        read_only: false
     environment:
       TZ: "Americas/New_York"
       ALLOWED_NETWORKS: 10.42.100.0/24,10.42.101.0/24

resources/docker-compose/scipio.yaml.j2

@@ -3,6 +3,7 @@ version: "{{ omni_compose_version | string }}"
 
 
 x-global-env: &globalenv
+  SCIPIO_SECRET_KEY: {{ omni_compose_app_secrets.scipio.application_key }}
   SCIPIO_DB_BACKEND: MARIA
   SCIPIO_DB_HOST: database
   SCIPIO_DB_PORT: "3306"
@@ -12,6 +13,7 @@ x-global-env: &globalenv
   SCIPIO_LOG_LEVEL: debug
   SCIPIO_LOG_RETENTION: "864000"
   SCIPIO_LOG_BACKEND: redis
+  SCIPIO_LOG_REDIS_SCHEMA: "0"
   SCIPIO_LOG_REDIS_HOSTNAME: cache
   SCIPIO_PHANTOM_FEED: https://blog.tipranks.com/feed/
   SCIPIO_PHANTOM_HANDLER: tipranks
@@ -65,6 +67,26 @@ services:
     deploy:
       replicas: 1
 
+  api:
+    image: scipio:{{ omni_compose_apps.scipio.versions.api | default(omni_compose_apps.scipio.versions.default) }}
+    hostname: scipio-api
+    depends_on:
+      - database
+      - cache
+    networks:
+      - scipio
+    ports:
+      - published: {{ omni_compose_apps.scipio.published.ports.8080 }}
+        target: 8080
+        protocol: tcp
+        mode: ingress
+    environment:
+      <<: *globalenv
+      SCIPIO_LOG_SOURCE: api
+    command: --api
+    deploy:
+      replicas: 1
+
   phantom:
     image: scipio:{{ omni_compose_apps.scipio.versions.phantom | default(omni_compose_apps.scipio.versions.default) }}
     hostname: scipio-phantom

resources/nextcloud-mariadb.cnf

@@ -0,0 +1,9 @@
+# https://docs.nextcloud.com/server/21/admin_manual/installation/server_tuning.html#using-mariadb-mysql-instead-of-sqlite
+# https://github.com/owncloud/core/issues/20967#issuecomment-205474772
+[mysqld]
+innodb_buffer_pool_size = 1G
+innodb_buffer_pool_instance = 1
+innodb_flush_log_at_trx_commit = 2
+innodb_log_buffer_size = 32M
+innodb_max_dirty_pages_pct = 90
+innodb_io_capacity=4000

resources/nextcloud-php-fpm.ini

@@ -0,0 +1,15 @@
+; https://docs.nextcloud.com/server/21/admin_manual/installation/server_tuning.html#tune-php-fpm
+pm = dynamic
+pm.max_children = 120
+pm.start_servers = 12
+pm.min_spare_servers = 6
+pm.max_spare_servers = 18
+
+; https://github.com/phpredis/phpredis#php-session-handler
+session.save_handler = redis
+session.save_path = "tcp://cache:6379?weight=1"
+
+; https://docs.nextcloud.com/server/21/admin_manual/configuration_server/caching_configuration.html#id2
+redis.session.locking_enabled=1
+redis.session.lock_retries=-1
+redis.session.lock_wait_time=10000

resources/nginx/nextcloud-proxy.conf

@@ -0,0 +1,88 @@
+server {
+        listen 80;
+        root /usr/share/nginx/nextcloud;
+        index index.php index.html index.htm /index.php$request_uri;
+
+        client_max_body_size 4G;
+        fastcgi_buffers 64 4k;
+
+        gzip on;
+        gzip_vary on;
+        gzip_comp_level 4;
+        gzip_min_length 256;
+        gzip_proxied expired no-cache no-store private no_last_modified no_etag auth;
+        gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy;
+
+        add_header Referrer-Policy                      "no-referrer"   always;
+        add_header X-Content-Type-Options               "nosniff"       always;
+        add_header X-Download-Options                   "noopen"        always;
+        add_header X-Frame-Options                      "SAMEORIGIN"    always;
+        add_header X-Permitted-Cross-Domain-Policies    "none"          always;
+        add_header X-Robots-Tag                         "none"          always;
+        add_header X-XSS-Protection                     "1; mode=block" always;
+
+        # Remove X-Powered-By, which is an information leak
+        fastcgi_hide_header X-Powered-By;
+
+        location = / {
+            if ( $http_user_agent ~ ^DavClnt ) {
+                return 302 /remote.php/webdav/$is_args$args;
+            }
+        }
+
+        location = /robots.txt {
+            allow all;
+            log_not_found off;
+            access_log off;
+        }
+
+        location ^~ /.well-known {
+            location = /.well-known/carddav  { return 301 /remote.php/dav/; }
+            location = /.well-known/caldav   { return 301 /remote.php/dav/; }
+            location ^~ /.well-known         { return 301 /index.php$uri; }
+            try_files $uri $uri/ =404;
+        }
+
+        location ~ ^/(?:build|tests|config|lib|3rdparty|templates|data)(?:$|/) { return 404; }
+        location ~ ^/(?:\.|autotest|occ|issue|indie|db_|console) { return 404; }
+
+        location ~ [^/]\.php(/|$) {
+            fastcgi_split_path_info ^(.+?\.php)(/.*)$;
+            set $path_info $fastcgi_path_info;
+
+            try_files $fastcgi_script_name =404;
+
+            include fastcgi_params;
+            fastcgi_intercept_errors on;
+            fastcgi_request_buffering off;
+
+            fastcgi_param  SCRIPT_FILENAME          /var/www/html/$fastcgi_script_name;
+            fastcgi_param  PATH_INFO                $path_info;
+            fastcgi_param  DOCUMENT_ROOT            /var/www/html/;
+            fastcgi_param  modHeadersAvailable      true;
+            fastcgi_param  front_controller_active  true;
+            fastcgi_param  HTTPS                    $https;
+            fastcgi_param  REDIRECT_STATUS          200;
+            # Mitigate https://httpoxy.org/ vulnerabilities
+            fastcgi_param  HTTP_PROXY               "";
+
+            fastcgi_pass server:9000;
+        }
+
+        location ~ \.(?:css|js|svg|gif)$ {
+            try_files $uri /index.php$request_uri;
+            expires 6M;
+            access_log off;
+        }
+
+        location ~ \.woff2?$ {
+            try_files $uri /index.php$request_uri;
+            expires 7d;
+            access_log off;
+        }
+
+      location / {
+          try_files $uri $uri/ /index.php$request_uri;
+      }
+
+}

tasks/docker/build.yml

@@ -1,5 +1,26 @@
 ---
-- name: Clone repositories
+- name: Download source
+  block:
+    - name: Clone repositories
+      when: item.value.build is defined
+      git:
+        repo: "{{ item.value.build.repository }}"
+        dest: /tmp/{{ item.key }}
+        version: "{{ item.value.build.version }}"
+        accept_hostkey: true
+      loop: "{{ omni_compose_apps | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
+  rescue:
+    - name: Remove existing repository downloads
+      file:
+        path: /tmp/{{ item.key }}
+        state: absent
+      loop: "{{ omni_compose_apps | dict2items }}"
+      loop_control:
+        label: "{{ item.key }}"
+
+    - name: Clone repositories
       when: item.value.build is defined
       git:
         repo: "{{ item.value.build.repository }}"

vars/accounts.yaml

@@ -77,3 +77,8 @@ omni_users:
     uid: 1291
     targets: [datastore]
     svc: true
+
+  - name: mech_nextcloud
+    uid: 1290
+    targets: [datastore]
+    svc: true

vars/applications.yaml

@@ -17,9 +17,8 @@ omni_compose_apps:
       internal: 192.168.104.0/24
       external: 192.168.105.0/24
     versions:
-      default: 1.36.1
+      default: 1.40.0
-      web: 2.15.1
+      web: 2.19.0
-      attachments: 1.34.0
 
   gitea:
     datastore: /appdata/gitea
@@ -34,7 +33,7 @@ omni_compose_apps:
     networks:
       main: 192.168.103.0/24
     versions:
-      default: 1.11.5
+      default: 1.14.1
 
   minecraft:
     datastore: /appdata/minecraft
@@ -48,9 +47,10 @@ omni_compose_apps:
       main: 192.168.102.0/24
     versions:
       main: latest
-      server: 1.15.2
+      server: 1.16.5
 
   plex:
+    force_clean: true
     datastore: /appdata/plex
     account:
       name: mech_plex
@@ -93,7 +93,7 @@ omni_compose_apps:
   scipio:
     build:
       repository: git@github.com:tjyork/Scipio.git
-      version: 0.6.0
+      version: 1.1.2
     datastore: /appdata/scipio
     account:
       name: mech_scipio
@@ -106,7 +106,30 @@ omni_compose_apps:
     networks:
       main: 192.168.106.0/24
     versions:
-      default: 0.8.0
+      default: 1.1.2
       database: "10"
       cache: "6.2"
-      dashboard: latest
+
+  nextcloud:
+    datastore: /appdata/nextcloud
+    account:
+      name: mech_nextcloud
+      uid: 1290
+    published:
+      host: nxc.enp.one
+      ports:
+        80: 8082
+    networks:
+      main: 192.168.107.0/24
+    versions:
+      proxy: latest
+      server: 21.0.1-fpm
+      database: "10"
+      cache: "6.2"
+    assets:
+      - src: nginx/nextcloud-proxy.conf
+        name: proxy/nextcloud.conf
+      - src: nextcloud-php-fpm.ini
+        name: config/php.ini
+      - src: nextcloud-mariadb.cnf
+        name: proxy/nextcloud.cnf