From 686d1749e9415d5bcf0c1ac84706e805c03e7a9c Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 15:27:51 +0200 Subject: [PATCH 01/19] add caching rules, tell client caching for 1 year, but no caching for index.html --- .../sites-available/gradido.conf.ssl.template | 16 ++++++++++++++++ .../nginx/sites-available/gradido.conf.template | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 294e9f8a0..40d93ad9a 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -51,6 +51,12 @@ server { rewrite ^/vue/?(.*)$ /$1 permanent; } + # caching rules for assets + location ~* \.(?:js|css|woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + # Frontend (default) location / { @@ -59,6 +65,11 @@ server { root $PROJECT_ROOT/frontend/build/; index index.html; try_files $uri $uri/ /index.html = 404; + + # don't cache index.html + add_header Cache-Control "no-cache, no-store, must-revalidate"; + add_header Pragma "no-cache"; + add_header Expires 0; access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; @@ -119,6 +130,11 @@ server { index index.html; try_files $uri $uri/ /index.html = 404; + # don't cache index.html + add_header Cache-Control "no-cache, no-store, must-revalidate"; + add_header Pragma "no-cache"; + add_header Expires 0; + access_log $GRADIDO_LOG_PATH/nginx-access.admin.log gradido_log; error_log $GRADIDO_LOG_PATH/nginx-error.admin.log warn; } diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 7bd28b228..ef63c42a4 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -36,6 +36,12 @@ server { rewrite ^/vue/?(.*)$ /$1 permanent; } + # caching rules for assets + location ~* \.(?:js|css|woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + # Frontend (default) location / { #limit_req zone=frontend burst=40 nodelay; @@ -44,6 +50,11 @@ server { index index.html; try_files $uri $uri/ /index.html = 404; + # don't cache index.html + add_header Cache-Control "no-cache, no-store, must-revalidate"; + add_header Pragma "no-cache"; + add_header Expires 0; + access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; } @@ -103,6 +114,11 @@ server { index index.html; try_files $uri $uri/ /index.html = 404; + # don't cache index.html + add_header Cache-Control "no-cache, no-store, must-revalidate"; + add_header Pragma "no-cache"; + add_header Expires 0; + access_log $GRADIDO_LOG_PATH/nginx-access.admin.log gradido_log; error_log $GRADIDO_LOG_PATH/nginx-error.admin.log warn; } From 830deb9b44f261f290ee8248517ae0d885e32006 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:07:17 +0200 Subject: [PATCH 02/19] adjust request limits, log requests limits --- .../nginx/common/limit_requests.conf | 4 +-- .../bare_metal/nginx/conf.d/logging.conf | 4 ++- .../gradido-federation.conf.template | 4 +-- .../sites-available/gradido.conf.ssl.template | 35 +++++++++++-------- .../sites-available/gradido.conf.template | 34 +++++++++++------- 5 files changed, 49 insertions(+), 32 deletions(-) diff --git a/deployment/bare_metal/nginx/common/limit_requests.conf b/deployment/bare_metal/nginx/common/limit_requests.conf index c9501fd64..022a6d3a3 100644 --- a/deployment/bare_metal/nginx/common/limit_requests.conf +++ b/deployment/bare_metal/nginx/common/limit_requests.conf @@ -1,4 +1,4 @@ -limit_req_zone $binary_remote_addr zone=frontend:20m rate=5r/s; -limit_req_zone $binary_remote_addr zone=backend:25m rate=15r/s; +limit_req_zone $binary_remote_addr zone=frontend:20m rate=15r/s; +limit_req_zone $binary_remote_addr zone=backend:25m rate=20r/s; limit_req_zone $binary_remote_addr zone=api:5m rate=30r/s; limit_conn_zone $binary_remote_addr zone=addr:10m; \ No newline at end of file diff --git a/deployment/bare_metal/nginx/conf.d/logging.conf b/deployment/bare_metal/nginx/conf.d/logging.conf index a76e8fae7..b8ed225a8 100644 --- a/deployment/bare_metal/nginx/conf.d/logging.conf +++ b/deployment/bare_metal/nginx/conf.d/logging.conf @@ -1,4 +1,6 @@ log_format gradido_log '$remote_addr - $remote_user [$time_local] ' '"$request_method $status $request_uri"' ' "$http_referer" "$http_user_agent"' - ' $server_protocol $body_bytes_sent $request_time'; \ No newline at end of file + ' $server_protocol $body_bytes_sent $request_time' + ' limit status: $limit_req_status' + ; \ No newline at end of file diff --git a/deployment/bare_metal/nginx/sites-available/gradido-federation.conf.template b/deployment/bare_metal/nginx/sites-available/gradido-federation.conf.template index cf5f53b25..5123deb5e 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido-federation.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido-federation.conf.template @@ -1,7 +1,7 @@ location /api/$FEDERATION_APIVERSION { - #limit_req zone=api burst=60 nodelay; - #limit_conn addr 30; + limit_req zone=api burst=60 nodelay; + limit_conn addr 30; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 40d93ad9a..c0791f169 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -25,8 +25,9 @@ server { include /etc/nginx/common/protect_add_header.conf; # protect from slow loris - #client_body_timeout 10s; - #client_header_timeout 10s; + client_body_timeout 10s; + client_header_timeout 10s; + send_timeout 10s # protect from range attack (in http header) if ($http_range ~ "d{9,}") { @@ -52,16 +53,22 @@ server { } # caching rules for assets - location ~* \.(?:js|css|woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; try_files $uri =404; } # Frontend (default) location / { - - #limit_req zone=frontend burst=40 nodelay; - #limit_conn addr 40; + limit_req zone=frontend burst=150 nodelay; + limit_conn addr 60; root $PROJECT_ROOT/frontend/build/; index index.html; try_files $uri $uri/ /index.html = 404; @@ -77,8 +84,8 @@ server { # Backend location /graphql { - #limit_req zone=backend burst=10 nodelay; - #limit_conn addr 10; + limit_req zone=backend burst=40 nodelay; + limit_conn addr 20; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; @@ -95,8 +102,8 @@ server { # Backend webhooks location /hook { - #limit_req zone=backend burst=10; - #limit_conn addr 10; + limit_req zone=backend burst=20 nodelay; + limit_conn addr 20; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; @@ -113,8 +120,8 @@ server { # Webhook reverse proxy location /hooks/ { - #limit_req zone=backend burst=10; - #limit_conn addr 10; + limit_req zone=backend burst=20 nodelay; + limit_conn addr 20; proxy_pass http://127.0.0.1:9000/hooks/; access_log $GRADIDO_LOG_PATH/nginx-access.hooks.log gradido_log; @@ -123,8 +130,8 @@ server { # Admin Frontend location /admin { - #limit_req zone=frontend burst=30 nodelay; - #limit_conn addr 40; + limit_req zone=frontend burst=30 nodelay; + limit_conn addr 20; rewrite ^/admin/(.*)$ /$1 break; root $PROJECT_ROOT/admin/build/; index index.html; diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index ef63c42a4..68fc2edb0 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -10,8 +10,9 @@ server { include /etc/nginx/common/protect_add_header.conf; # protect from slow loris - #client_body_timeout 10s; - #client_header_timeout 10s; + client_body_timeout 10s; + client_header_timeout 10s; + send_timeout 10s # protect from range attack (in http header) if ($http_range ~ "d{9,}") { @@ -37,15 +38,22 @@ server { } # caching rules for assets - location ~* \.(?:js|css|woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; try_files $uri =404; } # Frontend (default) location / { - #limit_req zone=frontend burst=40 nodelay; - #limit_conn addr 40; + limit_req zone=frontend burst=150 nodelay; + limit_conn addr 60; root $PROJECT_ROOT/frontend/build/; index index.html; try_files $uri $uri/ /index.html = 404; @@ -61,8 +69,8 @@ server { # Backend location /graphql { - #limit_req zone=backend burst=10 nodelay; - #limit_conn addr 10; + limit_req zone=backend burst=40 nodelay; + limit_conn addr 20; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; @@ -79,8 +87,8 @@ server { # Backend webhooks location /hook { - #limit_req zone=backend burst=10; - #limit_conn addr 10; + limit_req zone=backend burst=20 nodelay; + limit_conn addr 20; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; @@ -97,8 +105,8 @@ server { # Webhook reverse proxy location /hooks/ { - #limit_req zone=backend burst=10; - #limit_conn addr 10; + limit_req zone=backend burst=20 nodelay; + limit_conn addr 20; proxy_pass http://127.0.0.1:9000/hooks/; access_log $GRADIDO_LOG_PATH/nginx-access.hooks.log gradido_log; @@ -107,8 +115,8 @@ server { # Admin Frontend location /admin { - #limit_req zone=frontend burst=30 nodelay; - #limit_conn addr 40; + limit_req zone=frontend burst=30 nodelay; + limit_conn addr 20; rewrite ^/admin/(.*)$ /$1 break; root $PROJECT_ROOT/admin/build/; index index.html; From 3a4a246ca87d6aa29d192b9597f62467a3a2a985 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:14:44 +0200 Subject: [PATCH 03/19] add missing ; --- .../bare_metal/nginx/sites-available/gradido.conf.ssl.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index c0791f169..1f27a93f3 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -27,7 +27,7 @@ server { # protect from slow loris client_body_timeout 10s; client_header_timeout 10s; - send_timeout 10s + send_timeout 10s; # protect from range attack (in http header) if ($http_range ~ "d{9,}") { From c57473cf953c0ab93c9f59249b916da1d7a4ffb8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:17:14 +0200 Subject: [PATCH 04/19] fix bug in deploy script --- deployment/bare_metal/start.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index 37458b540..a812588ba 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -128,9 +128,8 @@ nginx_restart() { sudo systemctl start nginx } } -nginx_restart ln -sf $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default - +nginx_restart # helper functions log_step() { From 1a423552360699a18a98603b7601c7bff5b15383 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:18:02 +0200 Subject: [PATCH 05/19] fix typo --- deployment/bare_metal/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index a812588ba..17ec8f42d 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -128,7 +128,7 @@ nginx_restart() { sudo systemctl start nginx } } -ln -sf $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default +ln -s $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default nginx_restart # helper functions From f1870fe2f2a97feff439df19b547723c549d6cb8 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:20:48 +0200 Subject: [PATCH 06/19] fix --- deployment/bare_metal/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index 17ec8f42d..a812588ba 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -128,7 +128,7 @@ nginx_restart() { sudo systemctl start nginx } } -ln -s $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default +ln -sf $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default nginx_restart # helper functions From cbd7910f8b9077a21f692a121c870601110ed90d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:26:10 +0200 Subject: [PATCH 07/19] add logging to new routes --- .../nginx/sites-available/gradido.conf.ssl.template | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 1f27a93f3..864e7343a 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -58,11 +58,17 @@ server { # keep assets for a week add_header Cache-Control "public, max-age=604800"; try_files $uri =404; + + access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; + error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; try_files $uri =404; + + access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; + error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; } # Frontend (default) From 688906a2815fdc7b2df6b5a02a6e59142e5479c2 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:34:06 +0200 Subject: [PATCH 08/19] remove wrong line in nginx config --- .../nginx/sites-available/gradido.conf.ssl.template | 10 +--------- .../nginx/sites-available/gradido.conf.template | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 864e7343a..425abd69a 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -57,18 +57,10 @@ server { location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { # keep assets for a week add_header Cache-Control "public, max-age=604800"; - try_files $uri =404; - - access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; - error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; - try_files $uri =404; - - access_log $GRADIDO_LOG_PATH/nginx-access.frontend.log gradido_log; - error_log $GRADIDO_LOG_PATH/nginx-error.frontend.log warn; } # Frontend (default) @@ -77,7 +69,7 @@ server { limit_conn addr 60; root $PROJECT_ROOT/frontend/build/; index index.html; - try_files $uri $uri/ /index.html = 404; + try_files $uri $uri/ /index.html = 404; # don't cache index.html add_header Cache-Control "no-cache, no-store, must-revalidate"; diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 68fc2edb0..1e46318ee 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -42,12 +42,10 @@ server { location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { # keep assets for a week add_header Cache-Control "public, max-age=604800"; - try_files $uri =404; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; - try_files $uri =404; } # Frontend (default) From b48d47c5ee40600012636d863932f1c327d8a8d7 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:38:26 +0200 Subject: [PATCH 09/19] try with included in frontend location block --- .../sites-available/gradido.conf.ssl.template | 27 ++++++++++--------- .../sites-available/gradido.conf.template | 25 +++++++++-------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 425abd69a..c2fe889b8 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -52,24 +52,27 @@ server { rewrite ^/vue/?(.*)$ /$1 permanent; } - # caching rules for assets - # static assets - location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { - # keep assets for a week - add_header Cache-Control "public, max-age=604800"; - } - # hashed assets - location ~* \.(?:js|css|json)$ { - add_header Cache-Control "public, max-age=31536000, immutable"; - } - # Frontend (default) location / { limit_req zone=frontend burst=150 nodelay; limit_conn addr 60; root $PROJECT_ROOT/frontend/build/; index index.html; - try_files $uri $uri/ /index.html = 404; + + # caching rules for assets + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + + try_files $uri $uri/ /index.html = 404; # don't cache index.html add_header Cache-Control "no-cache, no-store, must-revalidate"; diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 1e46318ee..05d5719dd 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -37,23 +37,26 @@ server { rewrite ^/vue/?(.*)$ /$1 permanent; } - # caching rules for assets - # static assets - location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { - # keep assets for a week - add_header Cache-Control "public, max-age=604800"; - } - # hashed assets - location ~* \.(?:js|css|json)$ { - add_header Cache-Control "public, max-age=31536000, immutable"; - } - # Frontend (default) location / { limit_req zone=frontend burst=150 nodelay; limit_conn addr 60; root $PROJECT_ROOT/frontend/build/; index index.html; + + # caching rules for assets + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + try_files $uri $uri/ /index.html = 404; # don't cache index.html From 7ca419b3202efc8ab60e9a2e10cc4d644ba2b0fd Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:42:46 +0200 Subject: [PATCH 10/19] cache admin also --- .../sites-available/gradido.conf.ssl.template | 14 ++++++++++++++ .../nginx/sites-available/gradido.conf.template | 16 +++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index c2fe889b8..316f93da5 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -136,6 +136,20 @@ server { rewrite ^/admin/(.*)$ /$1 break; root $PROJECT_ROOT/admin/build/; index index.html; + + # caching rules for assets + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + try_files $uri $uri/ /index.html = 404; # don't cache index.html diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 05d5719dd..9b2331fa2 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -56,7 +56,7 @@ server { add_header Cache-Control "public, max-age=31536000, immutable"; try_files $uri =404; } - + try_files $uri $uri/ /index.html = 404; # don't cache index.html @@ -121,6 +121,20 @@ server { rewrite ^/admin/(.*)$ /$1 break; root $PROJECT_ROOT/admin/build/; index index.html; + + # caching rules for assets + # static assets + location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { + # keep assets for a week + add_header Cache-Control "public, max-age=604800"; + try_files $uri =404; + } + # hashed assets + location ~* \.(?:js|css|json)$ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + try_files $uri $uri/ /index.html = 404; # don't cache index.html From 30e22db650785a5b8ecd2b7ea81a44180d401f1b Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Wed, 21 May 2025 16:57:46 +0200 Subject: [PATCH 11/19] without try files --- .../nginx/sites-available/gradido.conf.ssl.template | 4 ++-- .../bare_metal/nginx/sites-available/gradido.conf.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index 316f93da5..ebcf6996a 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -142,12 +142,12 @@ server { location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { # keep assets for a week add_header Cache-Control "public, max-age=604800"; - try_files $uri =404; + # try_files $uri =404; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; - try_files $uri =404; + # try_files $uri =404; } try_files $uri $uri/ /index.html = 404; diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 9b2331fa2..2c9cd2324 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -127,12 +127,12 @@ server { location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { # keep assets for a week add_header Cache-Control "public, max-age=604800"; - try_files $uri =404; + # try_files $uri =404; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; - try_files $uri =404; + # try_files $uri =404; } try_files $uri $uri/ /index.html = 404; From f65dd5aba2fd96ae592c77277ea2168eb8755511 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 22 May 2025 07:01:52 +0200 Subject: [PATCH 12/19] try with alias in admin --- .../nginx/sites-available/gradido.conf.ssl.template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template index ebcf6996a..1eb01f09e 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.ssl.template @@ -133,8 +133,8 @@ server { location /admin { limit_req zone=frontend burst=30 nodelay; limit_conn addr 20; - rewrite ^/admin/(.*)$ /$1 break; - root $PROJECT_ROOT/admin/build/; + #rewrite ^/admin/(.*)$ /$1 break; + alias $PROJECT_ROOT/admin/build/; index index.html; # caching rules for assets @@ -142,12 +142,12 @@ server { location ~* \.(?:woff2?|ttf|otf|eot|jpg|jpeg|png|gif|svg|webp|ico)$ { # keep assets for a week add_header Cache-Control "public, max-age=604800"; - # try_files $uri =404; + try_files $uri =404; } # hashed assets location ~* \.(?:js|css|json)$ { add_header Cache-Control "public, max-age=31536000, immutable"; - # try_files $uri =404; + try_files $uri =404; } try_files $uri $uri/ /index.html = 404; From be8342183c3e16a073cb20c3e4c305e9fa5ccc0a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 22 May 2025 08:21:25 +0200 Subject: [PATCH 13/19] fix tty problem with docker --- deployment/bare_metal/start.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index 62406f7a0..f0457609b 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -101,7 +101,15 @@ TODAY=$(date +"%Y-%m-%d") # Create a new updating.html from the template \cp $SCRIPT_DIR/nginx/update-page/updating.html.template $UPDATE_HTML -# redirect all output of the script to the UPDATE_HTML and also have things on console +# store real console stream in fd 3 +if [ -e /dev/tty ]; then + # on normal systems + exec 3> /dev/tty +else + # on docker + exec 3> /proc/$$/fd/1 +fi +# redirect all output of the script to the UPDATE_HTML # TODO: this might pose a security risk exec > >(tee -a $UPDATE_HTML) 2>&1 @@ -109,9 +117,9 @@ exec > >(tee -a $UPDATE_HTML) 2>&1 echo 'Configuring nginx to serve the update-page' nginx_restart() { sudo /etc/init.d/nginx restart || { - echo -e "\e[33mwarn: nginx restart failed, will try to fix with 'sudo systemctl reset-failed nginx' and 'sudo systemctl start nginx'\e[0m" > /dev/tty + echo -e "\e[33mwarn: nginx restart failed, will try to fix with 'sudo systemctl reset-failed nginx' and 'sudo systemctl start nginx'\e[0m" >&3 sudo systemctl reset-failed nginx - sudo systemctl start nginx + sudo systemctl start nginx || sudo nginx -t } } ln -sf $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default @@ -120,22 +128,22 @@ nginx_restart # helper functions log_step() { local message="$1" - echo -e "\e[34m$message\e[0m" # > /dev/tty # blue in console + echo -e "\e[34m$message\e[0m" >&3 # blue in console echo "

$message

" >> "$UPDATE_HTML" # blue in html } log_error() { local message="$1" - echo -e "\e[31m$message\e[0m" # > /dev/tty # red in console + echo -e "\e[31m$message\e[0m" >&3 # red in console echo "$message" >> "$UPDATE_HTML" # red in html } log_warn() { local message="$1" - echo -e "\e[33m$message\e[0m" # > /dev/tty # orange in console + echo -e "\e[33m$message\e[0m" >&3 # orange in console echo "$message" >> "$UPDATE_HTML" # orange in html } log_success() { local message="$1" - echo -e "\e[32m$message\e[0m" # > /dev/tty # green in console + echo -e "\e[32m$message\e[0m" >&3 # green in console echo "

$message

" >> "$UPDATE_HTML" # green in html } From bd431cd7ea96ff442dd932bb5c3eb48f0a501bee Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 22 May 2025 08:42:20 +0200 Subject: [PATCH 14/19] fix check which wasn't working in docker started from ci --- deployment/bare_metal/start.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index f0457609b..bc3a56ad1 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -102,11 +102,11 @@ TODAY=$(date +"%Y-%m-%d") \cp $SCRIPT_DIR/nginx/update-page/updating.html.template $UPDATE_HTML # store real console stream in fd 3 -if [ -e /dev/tty ]; then - # on normal systems +if test -t 1; then + # stdout is a TTY - normal console exec 3> /dev/tty else - # on docker + # stdout is not a TTY - probably Docker or CI exec 3> /proc/$$/fd/1 fi # redirect all output of the script to the UPDATE_HTML From e78f8f897089f9a3fe2235f63b189d5c81b16e1d Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 22 May 2025 08:50:53 +0200 Subject: [PATCH 15/19] more infos if nginx failed --- deployment/bare_metal/start.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index bc3a56ad1..8403ce817 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -117,9 +117,12 @@ exec > >(tee -a $UPDATE_HTML) 2>&1 echo 'Configuring nginx to serve the update-page' nginx_restart() { sudo /etc/init.d/nginx restart || { - echo -e "\e[33mwarn: nginx restart failed, will try to fix with 'sudo systemctl reset-failed nginx' and 'sudo systemctl start nginx'\e[0m" >&3 + echo -e "\e[33mwarn: nginx restart failed\e[0m" >&3 + # run nginx -t to show problem but ignore exit code to prevent trap + { sudo nginx -t || true; } >&3 + echo -e "\e[33mwarn: will try to fix with 'sudo systemctl reset-failed nginx' and 'sudo systemctl start nginx'\e[0m" >&3 sudo systemctl reset-failed nginx - sudo systemctl start nginx || sudo nginx -t + sudo systemctl start nginx } } ln -sf $SCRIPT_DIR/nginx/sites-available/update-page.conf $SCRIPT_DIR/nginx/sites-enabled/default From 753b5b9d58ae640cb31bc6510b6469d97b64b07f Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Thu, 22 May 2025 08:56:57 +0200 Subject: [PATCH 16/19] finally find error, fix it --- .../bare_metal/nginx/sites-available/gradido.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/bare_metal/nginx/sites-available/gradido.conf.template b/deployment/bare_metal/nginx/sites-available/gradido.conf.template index 2c9cd2324..1f5ca2304 100644 --- a/deployment/bare_metal/nginx/sites-available/gradido.conf.template +++ b/deployment/bare_metal/nginx/sites-available/gradido.conf.template @@ -12,7 +12,7 @@ server { # protect from slow loris client_body_timeout 10s; client_header_timeout 10s; - send_timeout 10s + send_timeout 10s; # protect from range attack (in http header) if ($http_range ~ "d{9,}") { From b572e321bd3cea1f31b897fb05d89f7b72835d76 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 23 May 2025 19:36:46 +0200 Subject: [PATCH 17/19] add readme, start direct in production, less process count, more stable production --- deployment/bare_metal/Readme.md | 46 +++++++++++++++++++++++++++++++++ deployment/bare_metal/start.sh | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 deployment/bare_metal/Readme.md diff --git a/deployment/bare_metal/Readme.md b/deployment/bare_metal/Readme.md new file mode 100644 index 000000000..34d791e0f --- /dev/null +++ b/deployment/bare_metal/Readme.md @@ -0,0 +1,46 @@ +# Deployment for bare metal servers +This setup is designed for **bare metal servers**, offering maximum performance and reliability for Gradido deployments. However, it can also work on **virtual servers (VPS)** – if properly configured. + +## 🧠 Memory Considerations on VServers + +We have observed that some VServer providers apply **aggressive virtual memory constraints** or overcommit strategies that may cause **random crashes** of Node.js processes – even when total RAM appears sufficient. + +### Important Notes: + +- A single Node.js process may **allocate 10–12 GB of virtual memory** (VIRT), even if **real memory usage (RES)** stays below 200 MB. +- Some VPS environments **panic or kill processes** when virtual memory allocation exceeds certain invisible thresholds. + +## 🛡️ Rate Limiting (API Protection) + +This deployment includes built-in **rate limiting** for public-facing endpoints to prevent abuse and denial-of-service attacks. + +### 🔒 NGINX Rate & Connection Limits Overview + +| Path | Zone | Rate Limit | Burst | Max Connections | Notes | +|----------------------------|----------|----------------|-------|------------------|--------------------------------| +| `/` | frontend | 15 requests/s | 150 | 60 | Public frontend | +| `/admin` | frontend | 15 requests/s | 30 | 20 | Admin frontend | +| `/graphql` | backend | 20 requests/s | 40 | 20 | Main backend GraphQL API | +| `/hook` | backend | 20 requests/s | 20 | 20 | Internal backend webhooks | +| `/hooks/` | backend | 20 requests/s | 20 | 20 | Reverse proxy for webhooks | +| `/api/` | api | 30 requests/s | 60 | 30 | Federation GraphQL API | + +- ``: placeholder for federation api version +- All zones use `$binary_remote_addr` for client identification. +- `nodelay` ensures burst requests are not delayed (they are either accepted or rejected). +- Global connection zone: `limit_conn_zone $binary_remote_addr zone=addr:10m;` + +This setup helps protect public and internal interfaces from abuse, while ensuring smooth parallel access during high load periods (e.g., UI builds or cluster sync). + +These limits work like a traffic cop at each route: +- **Rate limits** (`limit_req`) define how many requests per second a single client can send. +- **Burst values** allow short spikes without blocking – like a temporary buffer. +- **Connection limits** (`limit_conn`) cap how many concurrent connections a single IP can keep open. + +Each route (frontend, backend, API, etc.) has its own configuration depending on its expected traffic pattern and sensitivity. For example: +- The public frontend allows higher bursts (many assets load at once). +- The GraphQL backend and admin interfaces are more tightly controlled. + +This ensures fairness, avoids accidental DoS scenarios, and keeps overall latency low, even under high usage. + + diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index f38621943..04218317e 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -287,7 +287,7 @@ else fi # start after building all to use up less ressources -pm2 start --name gradido-backend "turbo backend#start --env-mode=loose" -l $GRADIDO_LOG_PATH/pm2.backend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' +pm2 start --name gradido-backend "env TZ=UTC NODE_ENV=production node ./backend/build/index.js" -l $GRADIDO_LOG_PATH/pm2.backend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' #pm2 start --name gradido-frontend "yarn --cwd $PROJECT_ROOT/frontend start" -l $GRADIDO_LOG_PATH/pm2.frontend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' #pm2 start --name gradido-admin "yarn --cwd $PROJECT_ROOT/admin start" -l $GRADIDO_LOG_PATH/pm2.admin.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' pm2 save From e7376dd3fba664f54be8fe48a77badd3771c3e79 Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 23 May 2025 19:44:26 +0200 Subject: [PATCH 18/19] use correct path --- deployment/bare_metal/start.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index eb224559a..11754a3f4 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -297,9 +297,12 @@ else fi # start after building all to use up less ressources -pm2 start --name gradido-backend "env TZ=UTC NODE_ENV=production node ./backend/build/index.js" -l $GRADIDO_LOG_PATH/pm2.backend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' -#pm2 start --name gradido-frontend "yarn --cwd $PROJECT_ROOT/frontend start" -l $GRADIDO_LOG_PATH/pm2.frontend.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' -#pm2 start --name gradido-admin "yarn --cwd $PROJECT_ROOT/admin start" -l $GRADIDO_LOG_PATH/pm2.admin.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' +pm2 start --name gradido-backend \ + "env TZ=UTC NODE_ENV=production node ./build/index.js" \ + --cwd $PROJECT_ROOT/backend \ + -l $GRADIDO_LOG_PATH/pm2.backend.$TODAY.log \ + --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' + pm2 save if [ ! -z $FEDERATION_DHT_TOPIC ]; then pm2 start --name gradido-dht-node "turbo dht-node#start --env-mode=loose" -l $GRADIDO_LOG_PATH/pm2.dht-node.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' From e8750e06da6d23d0cab9579f61d0f5a1e451575a Mon Sep 17 00:00:00 2001 From: einhornimmond Date: Fri, 23 May 2025 19:46:58 +0200 Subject: [PATCH 19/19] do the same for dht-node and federation --- deployment/bare_metal/start.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deployment/bare_metal/start.sh b/deployment/bare_metal/start.sh index 11754a3f4..0eda8958a 100755 --- a/deployment/bare_metal/start.sh +++ b/deployment/bare_metal/start.sh @@ -305,7 +305,11 @@ pm2 start --name gradido-backend \ pm2 save if [ ! -z $FEDERATION_DHT_TOPIC ]; then - pm2 start --name gradido-dht-node "turbo dht-node#start --env-mode=loose" -l $GRADIDO_LOG_PATH/pm2.dht-node.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' + pm2 start --name gradido-dht-node \ + "env TZ=UTC NODE_ENV=production node ./build/index.js" \ + --cwd $PROJECT_ROOT/dht-node \ + -l $GRADIDO_LOG_PATH/pm2.dht-node.$TODAY.log \ + --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' pm2 save else log_step "=====================================================================" @@ -329,7 +333,11 @@ do log_step "====================================================" log_step " start $MODULENAME listening on port=$FEDERATION_PORT" log_step "====================================================" - pm2 start --name $MODULENAME "turbo federation#start --env-mode=loose" -l $GRADIDO_LOG_PATH/pm2.$MODULENAME.$TODAY.log --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' + pm2 start --name $MODULENAME \ + "env TZ=UTC NODE_ENV=production node ./build/index.js" \ + --cwd $PROJECT_ROOT/federation \ + -l $GRADIDO_LOG_PATH/pm2.$MODULENAME.$TODAY.log \ + --log-date-format 'YYYY-MM-DD HH:mm:ss.SSS' pm2 save done