Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

nixos/filebeat: Improve options #322425

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+147 −61
Draft

nixos/filebeat: Improve options #322425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions nixos/doc/manual/release-notes/rl-2411.section.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@

- The `xdg.portal.gtkUsePortal` option has been removed, as it had been deprecated for over 2 years. Using the `GTK_USE_PORTAL` environment variable in this manner is not intended nor encouraged by the GTK developers, but can still be done manually via `environment.sessionVariables`.

- Removed `filebeat` default output for elasticsearch to `127.0.0.1:9200` and improved nixos options for outputs.

## Other Notable Changes {#sec-release-24.11-notable-changes}

<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->
Expand Down
206 changes: 145 additions & 61 deletions nixos/modules/services/logging/filebeat.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
{ config, lib, utils, pkgs, ... }:
{
config,
lib,
utils,
pkgs,
...
}:

let
inherit (lib)
Expand All @@ -8,22 +14,22 @@ let
mkPackageOption
mkIf
mkOption
types;
types
;

cfg = config.services.filebeat;

json = pkgs.formats.json {};
json = pkgs.formats.json { };
in
{
meta.maintainers = with lib.maintainers; [ felbinger ];
options = {

services.filebeat = {

enable = mkEnableOption "filebeat";

package = mkPackageOption pkgs "filebeat" {
example = "filebeat7";
};
package = mkPackageOption pkgs "filebeat" { example = "filebeat7"; };

inputs = mkOption {
description = ''
Expand All @@ -41,23 +47,28 @@ in

See <https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html>.
'';
default = {};
type = types.attrsOf (types.submodule ({ name, ... }: {
freeformType = json.type;
options = {
type = mkOption {
type = types.str;
default = name;
description = ''
The input type.
default = { };
type = types.attrsOf (
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the module system will now properly merge this.

I think this could be similar to 490ba58 but not fully sure.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like it is working. I tested a couple configurations and just had one issue with an assertion. Also this is nothing I changed in this commit, it was just changed due to new formatting standard which has been introduced in #326407.

types.submodule (
{ name, ... }:
{
freeformType = json.type;
options = {
type = mkOption {
type = types.str;
default = name;
description = ''
The input type.

Look for the value after `type:` on
the individual input pages linked from
<https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html>.
'';
};
};
}));
Look for the value after `type:` on
the individual input pages linked from
<https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html>.
'';
};
};
}
)
);
example = literalExpression ''
{
journald.id = "everything"; # Only for filebeat7
Expand Down Expand Up @@ -91,23 +102,28 @@ in

See <https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html>.
'';
default = {};
type = types.attrsOf (types.submodule ({ name, ... }: {
freeformType = json.type;
options = {
module = mkOption {
type = types.str;
default = name;
description = ''
The name of the module.
default = { };
type = types.attrsOf (
types.submodule (
{ name, ... }:
{
freeformType = json.type;
options = {
module = mkOption {
type = types.str;
default = name;
description = ''
The name of the module.

Look for the value after `module:` on
the individual input pages linked from
<https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html>.
'';
};
};
}));
Look for the value after `module:` on
the individual input pages linked from
<https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html>.
'';
};
};
}
)
);
example = literalExpression ''
{
nginx = {
Expand All @@ -129,30 +145,56 @@ in
freeformType = json.type;

options = {

output.elasticsearch.hosts = mkOption {
type = with types; listOf str;
default = [ "127.0.0.1:9200" ];
example = [ "myEShost:9200" ];
output = mkOption {
type = types.nullOr (
types.attrsOf (
types.submodule {
freeformType = json.type;
options = {
enabled = mkEnableOption "<name>";
hosts = mkOption {
type = with types; listOf str;
default = [ ];
description = ''
The list of Elasticsearch/Logstash/Kafka/Redis nodes to connect to.
The events are distributed to these nodes in round robin order. If
one node becomes unreachable, the event is automatically sent to
another node. Each node can be defined as a URL or IP:PORT.
'';
};
};
}
)
);
default = {
elasticsearch = {
enable = true;
hosts = [ "127.0.0.1:9200" ];
};
};
example = {
elasticsearch.enable = false;
logstash = {
enable = true;
hosts = [ "myEShost:9200" ];
};
};
description = ''
The list of Elasticsearch nodes to connect to.

The events are distributed to these nodes in round
robin order. If one node becomes unreachable, the
event is automatically sent to another node. Each
Elasticsearch node can be defined as a URL or
IP:PORT. For example:
`http://192.15.3.2`,
`https://es.found.io:9230` or
`192.24.3.2:9300`. If no port is
specified, `9200` is used.
You configure Filebeat to write to a specific output by setting options in the
Outputs section of the `filebeat.yml` config file. Only a single output may be defined.

The following topics describe how to configure each supported output. If you've secured
the Elastic Stack, also read Secure for more about security-related configuration options.

For more information about the available outputs see
[elastic.co/guide/en/beats/filebeat/current/configuring-output.html](https://www.elastic.co/guide/en/beats/filebeat/current/configuring-output.html).
'';
};

filebeat = {
inputs = mkOption {
type = types.listOf json.type;
default = [];
default = [ ];
internal = true;
description = ''
Inputs specify how Filebeat locates and processes
Expand All @@ -163,7 +205,7 @@ in
};
modules = mkOption {
type = types.listOf json.type;
default = [];
default = [ ];
internal = true;
description = ''
Filebeat modules provide a quick way to get started
Expand All @@ -180,7 +222,7 @@ in
};
};
};
default = {};
default = { };
example = literalExpression ''
{
settings = {
Expand Down Expand Up @@ -215,6 +257,51 @@ in

config = mkIf cfg.enable {

assertions =
let
validOutputs = [
"elasticsearch"
"logstash"
"kafka"
"redis"
"file"
"console"
"discard"
];
validOutputsHosts = [
"elasticsearch"
"logstash"
"kafka"
"redis"
];

filterEnabled = attrset:
let
enabledNames = builtins.filter (name: attrset.${name}.enabled) (builtins.attrNames attrset);
in
builtins.listToAttrs (map (name: { name = name; value = attrset.${name}; }) enabledNames);

enabledOutputs = filterEnabled cfg.settings.output;
in
[
{
assertion = builtins.length (builtins.attrNames enabledOutputs) == 1;
message = "only one services.filebeat.settings.output can be configured";
}
{
assertion =
(builtins.length (builtins.attrNames enabledOutputs) != 1)
|| (builtins.elem (builtins.head (builtins.attrNames enabledOutputs)) validOutputs);
message = "services.filebeat.settings.output is invalid, choose one of: ${lib.concatStringsSep ", " validOutputs}";
}
{
assertion =
(builtins.length (builtins.attrNames enabledOutputs) != 1)
|| (builtins.elem (builtins.head (builtins.attrNames enabledOutputs)) validOutputsHosts);
message = "services.filebeat.settings.output.<name>.hosts can only be configured for: ${lib.concatStringsSep ", " validOutputsHosts}";
}
];

services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
services.filebeat.settings.filebeat.modules = attrValues cfg.modules;

Expand All @@ -229,10 +316,7 @@ in

umask u=rwx,g=,o=

${utils.genJqSecretsReplacementSnippet
cfg.settings
"/var/lib/filebeat/filebeat.yml"
}
${utils.genJqSecretsReplacementSnippet cfg.settings "/var/lib/filebeat/filebeat.yml"}
'';
ExecStart = ''
${cfg.package}/bin/filebeat -e \
Expand Down