From 6230936be276cd02e1e21e22ad0845582d098399 Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 1 Apr 2021 19:47:27 +0200 Subject: [PATCH 1/3] nixos/gitlab: Add options to control puma worker and threads numbers --- nixos/modules/services/misc/gitlab.nix | 64 +++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index f86653f3ead2..c2b8cbfbd765 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -652,6 +652,62 @@ in { description = "Extra configuration to merge into shell-config.yml"; }; + puma.workers = mkOption { + type = types.int; + default = 2; + apply = x: builtins.toString x; + description = '' + The number of worker processes Puma should spawn. This + controls the amount of parallel Ruby code can be + executed. GitLab recommends Number of CPU cores - + 1, but at least two. + + + + Each worker consumes quite a bit of memory, so + be careful when increasing this. + + + ''; + }; + + puma.threadsMin = mkOption { + type = types.int; + default = 0; + apply = x: builtins.toString x; + description = '' + The minimum number of threads Puma should use per + worker. + + + + Each thread consumes memory and contributes to Global VM + Lock contention, so be careful when increasing this. + + + ''; + }; + + puma.threadsMax = mkOption { + type = types.int; + default = 4; + apply = x: builtins.toString x; + description = '' + The maximum number of threads Puma should use per + worker. This limits how many threads Puma will automatically + spawn in response to requests. In contrast to workers, + threads will never be able to run Ruby code in parallel, but + give higher IO parallelism. + + + + Each thread consumes memory and contributes to Global VM + Lock contention, so be careful when increasing this. + + + ''; + }; + extraConfig = mkOption { type = types.attrs; default = {}; @@ -1145,7 +1201,13 @@ in { TimeoutSec = "infinity"; Restart = "on-failure"; WorkingDirectory = "${cfg.packages.gitlab}/share/gitlab"; - ExecStart = "${cfg.packages.gitlab.rubyEnv}/bin/puma -C ${cfg.statePath}/config/puma.rb -e production"; + ExecStart = concatStringsSep " " [ + "${cfg.packages.gitlab.rubyEnv}/bin/puma" + "-e production" + "-C ${cfg.statePath}/config/puma.rb" + "-w ${cfg.puma.workers}" + "-t ${cfg.puma.threadsMin}:${cfg.puma.threadsMax}" + ]; }; }; From 306fc0648b99682219049c95e2182f2c668f61e6 Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 8 Apr 2021 19:01:22 +0200 Subject: [PATCH 2/3] nixos/gitlab: Add Sidekiq MemoryKiller support Restart sidekiq automatically when it consumes too much memory. See https://docs.gitlab.com/ee/administration/operations/sidekiq_memory_killer.html for details. --- nixos/modules/services/misc/gitlab.nix | 53 +++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index c2b8cbfbd765..1240b3e69294 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -708,6 +708,49 @@ in { ''; }; + sidekiq.memoryKiller.enable = mkOption { + type = types.bool; + default = true; + description = '' + Whether the Sidekiq MemoryKiller should be turned + on. MemoryKiller kills Sidekiq when its memory consumption + exceeds a certain limit. + + See + for details. + ''; + }; + + sidekiq.memoryKiller.maxMemory = mkOption { + type = types.int; + default = 2000; + apply = x: builtins.toString (x * 1024); + description = '' + The maximum amount of memory, in MiB, a Sidekiq worker is + allowed to consume before being killed. + ''; + }; + + sidekiq.memoryKiller.graceTime = mkOption { + type = types.int; + default = 900; + apply = x: builtins.toString x; + description = '' + The time MemoryKiller waits after noticing excessive memory + consumption before killing Sidekiq. + ''; + }; + + sidekiq.memoryKiller.shutdownWait = mkOption { + type = types.int; + default = 30; + apply = x: builtins.toString x; + description = '' + The time allowed for all jobs to finish before Sidekiq is + killed forcefully. + ''; + }; + extraConfig = mkOption { type = types.attrs; default = {}; @@ -1049,7 +1092,11 @@ in { ] ++ optional (cfg.databaseHost == "") "postgresql.service"; wantedBy = [ "gitlab.target" ]; partOf = [ "gitlab.target" ]; - environment = gitlabEnv; + environment = gitlabEnv // (optionalAttrs cfg.sidekiq.memoryKiller.enable { + SIDEKIQ_MEMORY_KILLER_MAX_RSS = cfg.sidekiq.memoryKiller.maxMemory; + SIDEKIQ_MEMORY_KILLER_GRACE_TIME = cfg.sidekiq.memoryKiller.graceTime; + SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT = cfg.sidekiq.memoryKiller.shutdownWait; + }); path = with pkgs; [ postgresqlPackage git @@ -1061,13 +1108,15 @@ in { # Needed for GitLab project imports gnutar gzip + + procps # Sidekiq MemoryKiller ]; serviceConfig = { Type = "simple"; User = cfg.user; Group = cfg.group; TimeoutSec = "infinity"; - Restart = "on-failure"; + Restart = "always"; WorkingDirectory = "${cfg.packages.gitlab}/share/gitlab"; ExecStart="${cfg.packages.gitlab.rubyEnv}/bin/sidekiq -C \"${cfg.packages.gitlab}/share/gitlab/config/sidekiq_queues.yml\" -e production"; }; From 6389170b3927556ca3f2404b6525f2f57948419d Mon Sep 17 00:00:00 2001 From: talyz Date: Thu, 8 Apr 2021 19:03:44 +0200 Subject: [PATCH 3/3] nixos/gitlab: Set MALLOC_ARENA_MAX to "2" This should reduce memory fragmentation drastically and is recommended by both the Puma and the Sidekiq author. It's also the default value for Ruby deployments on Heroku. --- nixos/modules/services/misc/gitlab.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 1240b3e69294..fbd5d0185b03 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -155,6 +155,7 @@ let GITLAB_REDIS_CONFIG_FILE = pkgs.writeText "redis.yml" (builtins.toJSON redisConfig); prometheus_multiproc_dir = "/run/gitlab"; RAILS_ENV = "production"; + MALLOC_ARENA_MAX = "2"; }; gitlab-rake = pkgs.stdenv.mkDerivation {