From e22fc264bda400c8345368f0b07ca1e35f0f50f5 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Fri, 22 Nov 2024 17:05:47 +0100 Subject: [PATCH 1/3] Preflight check for usable space and cache size --- .../org/graylog/datanode/Configuration.java | 5 +- .../bindings/PreflightChecksBindings.java | 2 + .../OpensearchCacheSizePreflightCheck.java | 108 ++++++++++++++++++ ...OpensearchCacheSizePreflightCheckTest.java | 50 ++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java create mode 100644 data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java index 91065ce5dcfb..442cbecbe255 100644 --- a/data-node/src/main/java/org/graylog/datanode/Configuration.java +++ b/data-node/src/main/java/org/graylog/datanode/Configuration.java @@ -254,7 +254,10 @@ public class Configuration { @Parameter(value = "metrics_policy") private String metricsPolicy = "gl-datanode-metrics-ism"; - @Documentation(value = "Cache size for searchable snaphots") + /** + * @see Searchable snapshots + */ + @Documentation(value = "Cache size for searchable snaphots. ") @Parameter(value = "node_search_cache_size") private String searchCacheSize = "10gb"; diff --git a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java index f32c4b52d6e5..410c739cab9b 100644 --- a/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/bindings/PreflightChecksBindings.java @@ -25,6 +25,7 @@ import org.graylog.datanode.bootstrap.preflight.OpensearchBinPreflightCheck; import org.graylog.datanode.bootstrap.preflight.OpensearchConfigSync; import org.graylog.datanode.bootstrap.preflight.OpensearchDataDirCompatibilityCheck; +import org.graylog.datanode.bootstrap.preflight.OpensearchCacheSizePreflightCheck; import org.graylog.datanode.opensearch.CsrRequester; import org.graylog.datanode.opensearch.CsrRequesterImpl; import org.graylog2.bindings.providers.MongoConnectionProvider; @@ -47,6 +48,7 @@ protected void configure() { addPreflightCheck(DatanodeDirectoriesLockfileCheck.class); addPreflightCheck(OpenSearchPreconditionsCheck.class); addPreflightCheck(OpensearchDataDirCompatibilityCheck.class); + addPreflightCheck(OpensearchCacheSizePreflightCheck.class); // Mongodb is needed for legacy datanode storage, where we want to extract the certificate chain from // mongodb and store it in local keystore diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java new file mode 100644 index 000000000000..5d420eb0bbc6 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.bootstrap.preflight; + +import jakarta.annotation.Nonnull; +import jakarta.inject.Inject; +import org.apache.commons.io.FileUtils; +import org.graylog.datanode.Configuration; +import org.graylog2.bootstrap.preflight.PreflightCheck; +import org.graylog2.bootstrap.preflight.PreflightCheckException; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class OpensearchCacheSizePreflightCheck implements PreflightCheck { + + private final String configuredNodeSearchCacheSize; + private final Path opensearchDataLocation; + private final Function usableSpaceProvider; + + @Inject + public OpensearchCacheSizePreflightCheck(Configuration datanodeConfiguration) { + this(datanodeConfiguration.getNodeSearchCacheSize(), datanodeConfiguration.getOpensearchDataLocation(), OpensearchCacheSizePreflightCheck::getUsableSpace); + } + + public OpensearchCacheSizePreflightCheck(String cacheSize, Path opensearchDataLocation, Function usableSpaceProvider) { + this.configuredNodeSearchCacheSize = cacheSize; + this.opensearchDataLocation = opensearchDataLocation; + this.usableSpaceProvider = usableSpaceProvider; + } + + @Override + public void runCheck() throws PreflightCheckException { + final long usableSpace = usableSpaceProvider.apply(opensearchDataLocation); + final long cacheSize = toBytes(this.configuredNodeSearchCacheSize); + if (cacheSize >= usableSpace) { + final String usable = toHumanReadableSize(usableSpace); + throw new PreflightCheckException(""" + There is not enough usable space for the node search cache. Your system has only %s available. + Either decrease node_search_cache_size configuration or make sure that datanode has enough free disk space. + Current node_search_cache_size=%s""" + .formatted(usable, this.configuredNodeSearchCacheSize)); + + } + } + + @Nonnull + private static String toHumanReadableSize(long usableSpace) { + return FileUtils.byteCountToDisplaySize(usableSpace).replaceFirst("\\s", "").toLowerCase(Locale.ROOT); + } + + private static long getUsableSpace(Path opensearchDataLocation) { + final FileStore fileStore; + try { + fileStore = Files.getFileStore(opensearchDataLocation); + return fileStore.getUsableSpace(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static long toBytes(String cacheSize) { + long returnValue = -1; + Pattern patt = Pattern.compile("([\\d.]+)([GMK]B)", Pattern.CASE_INSENSITIVE); + Matcher matcher = patt.matcher(cacheSize); + Map powerMap = new HashMap<>(); + powerMap.put("GB", 3); + powerMap.put("MB", 2); + powerMap.put("KB", 1); + if (matcher.find()) { + String number = matcher.group(1); + int pow = powerMap.get(matcher.group(2).toUpperCase()); + BigDecimal bytes = new BigDecimal(number); + bytes = bytes.multiply(BigDecimal.valueOf(1024).pow(pow)); + returnValue = bytes.longValue(); + } + + if (returnValue == -1) { + throw new PreflightCheckException(String.format(Locale.ROOT, "Unexpected value %s of node_search_cache_size", cacheSize)); + } + + return returnValue; + } + +} diff --git a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java new file mode 100644 index 000000000000..13ade5f941a0 --- /dev/null +++ b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.bootstrap.preflight; + +import org.assertj.core.api.Assertions; +import org.graylog2.bootstrap.preflight.PreflightCheckException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +class OpensearchCacheSizePreflightCheckTest { + + @Test + void testIllegalCacheSizeValue(@TempDir Path temp) { + final OpensearchCacheSizePreflightCheck check = new OpensearchCacheSizePreflightCheck("10g", temp, path -> gbToBytes(5)); + Assertions.assertThatThrownBy(check::runCheck) + .isInstanceOf(PreflightCheckException.class) + .hasMessageContaining("Unexpected value 10g of node_search_cache_size"); + } + + @Test + void testCacheSizeErrors(@TempDir Path temp) { + final OpensearchCacheSizePreflightCheck check = new OpensearchCacheSizePreflightCheck("10gb", temp, path -> gbToBytes(5)); + Assertions.assertThatThrownBy(check::runCheck) + .isInstanceOf(PreflightCheckException.class) + .hasMessageContaining("There is not enough usable space for the node search cache"); + + + Assertions.assertThatNoException().isThrownBy(() -> new OpensearchCacheSizePreflightCheck("2gb", temp, path -> gbToBytes(15)).runCheck()); + } + + private static long gbToBytes(long gb) { + return gb * 1024 * 1024 * 1024; + } +} From c0fbb723b1b0a53b75aed9e3f5e5a3a81c331697 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Mon, 25 Nov 2024 08:31:46 +0100 Subject: [PATCH 2/3] add warning when search cache size occupies more than 80% of free disk space --- .../OpensearchCacheSizePreflightCheck.java | 18 ++++++++++++++---- .../OpensearchCacheSizePreflightCheckTest.java | 5 ++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java index 5d420eb0bbc6..de732c08e8ab 100644 --- a/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java +++ b/data-node/src/main/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheck.java @@ -22,6 +22,8 @@ import org.graylog.datanode.Configuration; import org.graylog2.bootstrap.preflight.PreflightCheck; import org.graylog2.bootstrap.preflight.PreflightCheckException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.math.BigDecimal; @@ -41,6 +43,8 @@ public class OpensearchCacheSizePreflightCheck implements PreflightCheck { private final Path opensearchDataLocation; private final Function usableSpaceProvider; + private static final Logger LOG = LoggerFactory.getLogger(OpensearchCacheSizePreflightCheck.class); + @Inject public OpensearchCacheSizePreflightCheck(Configuration datanodeConfiguration) { this(datanodeConfiguration.getNodeSearchCacheSize(), datanodeConfiguration.getOpensearchDataLocation(), OpensearchCacheSizePreflightCheck::getUsableSpace); @@ -56,17 +60,23 @@ public OpensearchCacheSizePreflightCheck(String cacheSize, Path opensearchDataLo public void runCheck() throws PreflightCheckException { final long usableSpace = usableSpaceProvider.apply(opensearchDataLocation); final long cacheSize = toBytes(this.configuredNodeSearchCacheSize); + final String usableHumanReadable = toHumanReadableSize(usableSpace); if (cacheSize >= usableSpace) { - final String usable = toHumanReadableSize(usableSpace); throw new PreflightCheckException(""" There is not enough usable space for the node search cache. Your system has only %s available. Either decrease node_search_cache_size configuration or make sure that datanode has enough free disk space. Current node_search_cache_size=%s""" - .formatted(usable, this.configuredNodeSearchCacheSize)); - + .formatted(usableHumanReadable, this.configuredNodeSearchCacheSize)); + } else if (percentageUsage(usableSpace, cacheSize) > 80.0) { + LOG.warn("Your system is running out of disk space. Current node_search_cache_size is configured to {} " + + "and your disk has only {} available.", this.configuredNodeSearchCacheSize, usableHumanReadable); } } + private double percentageUsage(long usableSpace, long cacheSize) { + return 100.0 / usableSpace * cacheSize; + } + @Nonnull private static String toHumanReadableSize(long usableSpace) { return FileUtils.byteCountToDisplaySize(usableSpace).replaceFirst("\\s", "").toLowerCase(Locale.ROOT); @@ -92,7 +102,7 @@ public static long toBytes(String cacheSize) { powerMap.put("KB", 1); if (matcher.find()) { String number = matcher.group(1); - int pow = powerMap.get(matcher.group(2).toUpperCase()); + int pow = powerMap.get(matcher.group(2).toUpperCase(Locale.ROOT)); BigDecimal bytes = new BigDecimal(number); bytes = bytes.multiply(BigDecimal.valueOf(1024).pow(pow)); returnValue = bytes.longValue(); diff --git a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java index 13ade5f941a0..2aea0d26562e 100644 --- a/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java +++ b/data-node/src/test/java/org/graylog/datanode/bootstrap/preflight/OpensearchCacheSizePreflightCheckTest.java @@ -39,9 +39,12 @@ void testCacheSizeErrors(@TempDir Path temp) { Assertions.assertThatThrownBy(check::runCheck) .isInstanceOf(PreflightCheckException.class) .hasMessageContaining("There is not enough usable space for the node search cache"); + } - + @Test + void testValidConfiguration(@TempDir Path temp) { Assertions.assertThatNoException().isThrownBy(() -> new OpensearchCacheSizePreflightCheck("2gb", temp, path -> gbToBytes(15)).runCheck()); + Assertions.assertThatNoException().isThrownBy(() -> new OpensearchCacheSizePreflightCheck("9gb", temp, path -> gbToBytes(10)).runCheck()); } private static long gbToBytes(long gb) { From 33a8585a4710bec3a85c9655e4ecd92946a372c5 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Mon, 25 Nov 2024 10:57:54 +0100 Subject: [PATCH 3/3] Added changelog --- changelog/unreleased/pr-21031.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/unreleased/pr-21031.toml diff --git a/changelog/unreleased/pr-21031.toml b/changelog/unreleased/pr-21031.toml new file mode 100644 index 000000000000..7234b939a9d5 --- /dev/null +++ b/changelog/unreleased/pr-21031.toml @@ -0,0 +1,4 @@ +type = "a" +message = "Added data node preflight check for usable disk space and node search cache size" + +pulls = ["21031"]