Merge pull request #317152 from jpds/quickwit-0.8.1

quickwit: 0.8.0 → 0.8.1, init module
This commit is contained in:
Yt 2024-06-06 01:24:58 +00:00 committed by GitHub
commit 6c50c9f872
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 423 additions and 34 deletions

@ -12,6 +12,8 @@
for LLMs. Available as [services.open-webui](#opt-services.open-webui.enable)
service.
- [Quickwit](https://quickwit.io), sub-second search & analytics engine on cloud storage. Available as [services.quickwit](options.html#opt-services.quickwit).
## Backward Incompatibilities {#sec-release-24.11-incompatibilities}
- `nginx` package no longer includes `gd` and `geoip` dependencies. For enabling it, override `nginx` package with the optionals `withImageFilter` and `withGeoIP`.

@ -1248,6 +1248,7 @@
./services/search/meilisearch.nix
./services/search/opensearch.nix
./services/search/qdrant.nix
./services/search/quickwit.nix
./services/search/sonic-server.nix
./services/search/typesense.nix
./services/security/aesmd.nix

@ -0,0 +1,190 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.quickwit;
settingsFormat = pkgs.formats.yaml {};
quickwitYml = settingsFormat.generate "quickwit.yml" cfg.settings;
usingDefaultDataDir = cfg.dataDir == "/var/lib/quickwit";
usingDefaultUserAndGroup = cfg.user == "quickwit" && cfg.group == "quickwit";
in
{
options.services.quickwit = {
enable = mkEnableOption "Quickwit";
package = lib.mkPackageOption pkgs "Quickwit" {
default = [ "quickwit" ];
};
settings = lib.mkOption {
type = lib.types.submodule {
freeformType = settingsFormat.type;
options."rest" = lib.mkOption {
default = {};
description = ''
Rest server configuration for Quickwit
'';
type = lib.types.submodule {
freeformType = settingsFormat.type;
options."listen_port" = lib.mkOption {
type = lib.types.port;
default = 7280;
description = ''
The port to listen on for HTTP REST traffic.
'';
};
};
};
options."grpc_listen_port" = lib.mkOption {
type = lib.types.port;
default = 7281;
description = ''
The port to listen on for gRPC traffic.
'';
};
options."listen_address" = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
description = ''
Listen address of Quickwit.
'';
};
options."version" = lib.mkOption {
type = lib.types.float;
default = 0.7;
description = ''
Configuration file version.
'';
};
};
default = {};
description = ''
Quickwit configuration.
'';
};
dataDir = lib.mkOption {
type = lib.types.path;
default = "/var/lib/quickwit";
apply = converge (removeSuffix "/");
description = ''
Data directory for Quickwit. If you change this, you need to
manually create the directory. You also need to create the
`quickwit` user and group, or change
[](#opt-services.quickwit.user) and
[](#opt-services.quickwit.group) to existing ones with
access to the directory.
'';
};
user = lib.mkOption {
type = lib.types.str;
default = "quickwit";
description = ''
The user Quickwit runs as. Should be left at default unless
you have very specific needs.
'';
};
group = lib.mkOption {
type = lib.types.str;
default = "quickwit";
description = ''
The group quickwit runs as. Should be left at default unless
you have very specific needs.
'';
};
extraFlags = lib.mkOption {
description = "Extra command line options to pass to Quickwit.";
default = [ ];
type = lib.types.listOf lib.types.str;
};
restartIfChanged = lib.mkOption {
type = lib.types.bool;
description = ''
Automatically restart the service on config change.
This can be set to false to defer restarts on a server or cluster.
Please consider the security implications of inadvertently running an older version,
and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
'';
default = true;
};
};
config = mkIf cfg.enable {
systemd.services.quickwit = {
description = "Quickwit";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
inherit (cfg) restartIfChanged;
environment = {
QW_DATA_DIR = cfg.dataDir;
};
serviceConfig = {
ExecStart = ''
${cfg.package}/bin/quickwit run --config ${quickwitYml} \
${escapeShellArgs cfg.extraFlags}
'';
User = cfg.user;
Group = cfg.group;
Restart = "on-failure";
DynamicUser = usingDefaultUserAndGroup && usingDefaultDataDir;
CapabilityBoundingSet = [ "" ];
DevicePolicy = "closed";
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
ProcSubset = "pid";
ProtectClock = true;
ProtectHome = true;
ProtectHostname = true;
ProtectControlGroups = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
ProtectSystem = "strict";
ReadWritePaths = [
"/var/lib/quickwit"
];
RestrictAddressFamilies = [
"AF_NETLINK"
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
# 1. allow a reasonable set of syscalls
"@system-service @resources"
# 2. and deny unreasonable ones
"~@privileged"
# 3. then allow the required subset within denied groups
"@chown"
];
} // (optionalAttrs (usingDefaultDataDir) {
StateDirectory = "quickwit";
StateDirectoryMode = "0700";
});
};
environment.systemPackages = [ cfg.package ];
};
}

@ -788,6 +788,7 @@ in {
qtile = handleTestOn ["x86_64-linux" "aarch64-linux"] ./qtile.nix {};
quake3 = handleTest ./quake3.nix {};
quicktun = handleTest ./quicktun.nix {};
quickwit = handleTest ./quickwit.nix {};
quorum = handleTest ./quorum.nix {};
rabbitmq = handleTest ./rabbitmq.nix {};
radarr = handleTest ./radarr.nix {};

31
nixos/tests/quickwit.nix Normal file

@ -0,0 +1,31 @@
import ./make-test-python.nix ({ lib, pkgs, ... }:
{
name = "quickwit";
meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
nodes = {
quickwit = { config, pkgs, ... }: {
services.quickwit.enable = true;
};
};
testScript =
''
quickwit.wait_for_unit("quickwit")
quickwit.wait_for_open_port(7280)
quickwit.wait_for_open_port(7281)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'version: ${pkgs.quickwit.version}'"
)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'transitioned to ready state'"
)
quickwit.log(quickwit.succeed(
"systemd-analyze security quickwit.service | grep -v ''"
))
'';
})

@ -8,4 +8,5 @@
api = import ./api.nix { inherit system pkgs; };
dnstap = import ./dnstap.nix { inherit system pkgs; };
nginx-clickhouse = import ./nginx-clickhouse.nix { inherit system pkgs; };
syslog-quickwit = import ./syslog-quickwit.nix { inherit system pkgs; };
}

@ -0,0 +1,155 @@
import ../make-test-python.nix ({ lib, pkgs, ... }:
# Based on https://quickwit.io/docs/log-management/send-logs/using-vector
{
name = "vector-syslog-quickwit";
meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
nodes = {
quickwit = { config, pkgs, ... }: {
environment.systemPackages = [ pkgs.jq ];
networking.firewall.allowedTCPPorts = [ 7280 ];
services.quickwit = {
enable = true;
settings = {
listen_address = "::";
};
};
};
syslog = { config, pkgs, ... }: {
services.vector = {
enable = true;
settings = {
sources = {
generate_syslog = {
type = "demo_logs";
format = "syslog";
interval = 0.5;
};
};
transforms = {
remap_syslog = {
inputs = ["generate_syslog"];
type = "remap";
source = ''
structured = parse_syslog!(.message)
.timestamp_nanos = to_unix_timestamp!(structured.timestamp, unit: "nanoseconds")
.body = structured
.service_name = structured.appname
.resource_attributes.source_type = .source_type
.resource_attributes.host.hostname = structured.hostname
.resource_attributes.service.name = structured.appname
.attributes.syslog.procid = structured.procid
.attributes.syslog.facility = structured.facility
.attributes.syslog.version = structured.version
.severity_text = if includes(["emerg", "err", "crit", "alert"], structured.severity) {
"ERROR"
} else if structured.severity == "warning" {
"WARN"
} else if structured.severity == "debug" {
"DEBUG"
} else if includes(["info", "notice"], structured.severity) {
"INFO"
} else {
structured.severity
}
.scope_name = structured.msgid
del(.message)
del(.timestamp)
del(.service)
del(.source_type)
'';
};
};
sinks = {
#emit_syslog = {
# inputs = ["remap_syslog"];
# type = "console";
# encoding.codec = "json";
#};
quickwit_logs = {
type = "http";
method = "post";
inputs = [ "remap_syslog" ];
encoding.codec = "json";
framing.method = "newline_delimited";
uri = "http://quickwit:7280/api/v1/otel-logs-v0_7/ingest";
};
};
};
};
};
};
testScript =
let
aggregationQuery = pkgs.writeText "aggregation-query.json" ''
{
"query": "*",
"max_hits": 0,
"aggs": {
"count_per_minute": {
"histogram": {
"field": "timestamp_nanos",
"interval": 60000000
},
"aggs": {
"severity_text_count": {
"terms": {
"field": "severity_text"
}
}
}
}
}
}
'';
in
''
quickwit.wait_for_unit("quickwit")
quickwit.wait_for_open_port(7280)
quickwit.wait_for_open_port(7281)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'transitioned to ready state'"
)
syslog.wait_for_unit("vector")
syslog.wait_until_succeeds(
"journalctl -o cat -u vector.service | grep 'Vector has started'"
)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'publish-new-splits'"
)
# Wait for logs to be generated
# Test below aggregates by the minute
syslog.sleep(60 * 2)
quickwit.wait_until_succeeds(
"curl -sSf -XGET http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search?query=severity_text:ERROR |"
+ " jq '.num_hits' | grep -v '0'"
)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'SearchRequest'"
)
quickwit.wait_until_succeeds(
"curl -sSf -XPOST -H 'Content-Type: application/json' http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search --data @${aggregationQuery} |"
+ " jq '.num_hits' | grep -v '0'"
)
quickwit.wait_until_succeeds(
"journalctl -o cat -u quickwit.service | grep 'count_per_minute'"
)
'';
})

@ -1240,7 +1240,7 @@ dependencies = [
[[package]]
name = "chitchat"
version = "0.8.0"
source = "git+https://github.com/quickwit-oss/chitchat.git?rev=f783620#f78362008b4b5522181c77e830f585c9d9e8b799"
source = "git+https://github.com/quickwit-oss/chitchat.git?rev=d039699#d03969982e1c199aa05cb8e4f86d8d83118057ff"
dependencies = [
"anyhow",
"async-trait",
@ -5441,7 +5441,7 @@ dependencies = [
[[package]]
name = "quickwit-actors"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5461,7 +5461,7 @@ dependencies = [
[[package]]
name = "quickwit-aws"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"async-trait",
"aws-config",
@ -5483,7 +5483,7 @@ dependencies = [
[[package]]
name = "quickwit-cli"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5541,7 +5541,7 @@ dependencies = [
[[package]]
name = "quickwit-cluster"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5570,7 +5570,7 @@ dependencies = [
[[package]]
name = "quickwit-codegen"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"futures",
@ -5587,7 +5587,7 @@ dependencies = [
[[package]]
name = "quickwit-codegen-example"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5614,7 +5614,7 @@ dependencies = [
[[package]]
name = "quickwit-common"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-speed-limit",
@ -5652,7 +5652,7 @@ dependencies = [
[[package]]
name = "quickwit-config"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"bytes",
@ -5686,7 +5686,7 @@ dependencies = [
[[package]]
name = "quickwit-control-plane"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5724,7 +5724,7 @@ dependencies = [
[[package]]
name = "quickwit-datetime"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"itertools 0.12.1",
@ -5738,7 +5738,7 @@ dependencies = [
[[package]]
name = "quickwit-directories"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5759,7 +5759,7 @@ dependencies = [
[[package]]
name = "quickwit-doc-mapper"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"base64 0.21.7",
@ -5794,7 +5794,7 @@ dependencies = [
[[package]]
name = "quickwit-index-management"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5826,7 +5826,7 @@ dependencies = [
[[package]]
name = "quickwit-indexing"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"arc-swap",
@ -5891,7 +5891,7 @@ dependencies = [
[[package]]
name = "quickwit-ingest"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5931,7 +5931,7 @@ dependencies = [
[[package]]
name = "quickwit-integration-tests"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"bytes",
@ -5964,7 +5964,7 @@ dependencies = [
[[package]]
name = "quickwit-jaeger"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -5998,7 +5998,7 @@ dependencies = [
[[package]]
name = "quickwit-janitor"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -6035,7 +6035,7 @@ dependencies = [
[[package]]
name = "quickwit-lambda"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"aws_lambda_events",
@ -6075,7 +6075,7 @@ dependencies = [
[[package]]
name = "quickwit-macros"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"proc-macro2",
"quote",
@ -6084,7 +6084,7 @@ dependencies = [
[[package]]
name = "quickwit-metastore"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -6127,7 +6127,7 @@ dependencies = [
[[package]]
name = "quickwit-opentelemetry"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -6151,7 +6151,7 @@ dependencies = [
[[package]]
name = "quickwit-proto"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -6188,7 +6188,7 @@ dependencies = [
[[package]]
name = "quickwit-query"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"base64 0.21.7",
@ -6214,7 +6214,7 @@ dependencies = [
[[package]]
name = "quickwit-rest-client"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"bytes",
@ -6238,7 +6238,7 @@ dependencies = [
[[package]]
name = "quickwit-search"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"assert-json-diff 2.0.2",
@ -6290,7 +6290,7 @@ dependencies = [
[[package]]
name = "quickwit-serve"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"assert-json-diff 2.0.2",
@ -6359,7 +6359,7 @@ dependencies = [
[[package]]
name = "quickwit-storage"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"anyhow",
"async-trait",
@ -6407,7 +6407,7 @@ dependencies = [
[[package]]
name = "quickwit-telemetry"
version = "0.8.0"
version = "0.8.1"
dependencies = [
"async-trait",
"encoding_rs",

@ -2,6 +2,7 @@
, lib
, fetchFromGitHub
, rustPlatform
, nixosTests
, nix-update-script
, protobuf
, rust-jemalloc-sys
@ -10,7 +11,7 @@
let
pname = "quickwit";
version = "0.8.0";
version = "0.8.1";
in
rustPlatform.buildRustPackage rec {
inherit pname version;
@ -19,7 +20,7 @@ rustPlatform.buildRustPackage rec {
owner = "quickwit-oss";
repo = pname;
rev = "v${version}";
hash = "sha256-FZVGQfDuQYIdRnCsBZvXeLbJBdcLugZeHNm+kf6L9SY=";
hash = "sha256-B5U9nzXh6kj3/UnQzM3//h4hn9ippWHbeDMcMTP9XfM=";
};
postPatch = ''
@ -40,7 +41,7 @@ rustPlatform.buildRustPackage rec {
cargoLock = {
lockFile = ./Cargo.lock;
outputHashes = {
"chitchat-0.8.0" = "sha256-cjwKaBXoztYUXgnJvtFH+OSQU6tl2U3zKFWX324+9wo=";
"chitchat-0.8.0" = "sha256-6K2noPoFaDnOxQIEV1WbmVPfRGwlI/WS1OWSBH2qb1Q=";
"mrecordlog-0.4.0" = "sha256-9LIVs+BqK9FLSfHL3vm9LL+/FXIXJ6v617QLv4luQik=";
"ownedbytes-0.6.0" = "sha256-in18/NYYIgUiZ9sm8NgJlebWidRp34DR7AhOD1Nh0aw=";
"pulsar-5.0.2" = "sha256-j7wpsAro6x4fk3pvSL4fxLkddJFq8duZ7jDj0Edf3YQ=";
@ -53,7 +54,13 @@ rustPlatform.buildRustPackage rec {
PROTOC = "${protobuf}/bin/protoc";
PROTOC_INCLUDE = "${protobuf}/include";
passthru.updateScript = nix-update-script { };
passthru = {
tests = {
inherit (nixosTests) quickwit;
inherit (nixosTests.vector) syslog-quickwit;
};
updateScript = nix-update-script { };
};
checkFlags = [
# tries to make a network access
@ -72,6 +79,7 @@ rustPlatform.buildRustPackage rec {
"--skip=object_storage::s3_compatible_storage::tests::test_s3_compatible_storage_relative_path"
# flaky test
"--skip=actors::indexer::tests::test_indexer_triggers_commit_on_drained_mailbox"
"--skip=actors::indexer::tests::test_indexer_triggers_commit_on_timeout"
"--skip=actors::indexer::tests::test_indexer_partitioning"
"--skip=actors::indexing_pipeline::tests::test_merge_pipeline_does_not_stop_on_indexing_pipeline_failure"
"--skip=actors::indexer::tests::test_indexer_triggers_commit_on_target_num_docs"