From 877b79f9eb96a350f49750ded7395a79e7c18ed6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:21:22 +0200 Subject: [PATCH 01/25] Bump react-router-dom (#20191) Bumps [react-router-dom](https://github.com/remix-run/react-router/tree/HEAD/packages/react-router-dom) from 6.26.0 to 6.26.1. - [Release notes](https://github.com/remix-run/react-router/releases) - [Changelog](https://github.com/remix-run/react-router/blob/main/packages/react-router-dom/CHANGELOG.md) - [Commits](https://github.com/remix-run/react-router/commits/react-router-dom@6.26.1/packages/react-router-dom) --- updated-dependencies: - dependency-name: react-router-dom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog2-web-interface/packages/graylog-web-plugin/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graylog2-web-interface/packages/graylog-web-plugin/package.json b/graylog2-web-interface/packages/graylog-web-plugin/package.json index 5e94f85d2e01..a8b545c07f30 100644 --- a/graylog2-web-interface/packages/graylog-web-plugin/package.json +++ b/graylog2-web-interface/packages/graylog-web-plugin/package.json @@ -48,7 +48,7 @@ "react": "18.3.1", "react-dom": "18.2.0", "react-router-bootstrap": "0.26.3", - "react-router-dom": "6.26.0", + "react-router-dom": "6.26.1", "reflux": "0.2.13", "styled-components": "6.1.1", "typescript": "5.5.4", From 05401aba54b91ba765642a478fddd0e1d84268f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:22:10 +0200 Subject: [PATCH 02/25] Bump marked from 13.0.1 to 14.0.0 in /graylog2-web-interface (#20193) Bumps [marked](https://github.com/markedjs/marked) from 13.0.1 to 14.0.0. - [Release notes](https://github.com/markedjs/marked/releases) - [Changelog](https://github.com/markedjs/marked/blob/master/.releaserc.json) - [Commits](https://github.com/markedjs/marked/compare/v13.0.1...v14.0.0) --- updated-dependencies: - dependency-name: marked dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog2-web-interface/package.json | 2 +- graylog2-web-interface/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/graylog2-web-interface/package.json b/graylog2-web-interface/package.json index 8b7e56c5d9ad..3d0ca6778b29 100644 --- a/graylog2-web-interface/package.json +++ b/graylog2-web-interface/package.json @@ -71,7 +71,7 @@ "interactjs": "^1.4.0-rc.13", "leaflet": "^1.5.1", "lodash": "^4.17.4", - "marked": "^13.0.0", + "marked": "^14.0.0", "material-symbols": "^0.22.0", "md5": "^2.2.1", "moment-duration-format": "2.3.2", diff --git a/graylog2-web-interface/yarn.lock b/graylog2-web-interface/yarn.lock index 756793c9c66f..a7a157914f6c 100644 --- a/graylog2-web-interface/yarn.lock +++ b/graylog2-web-interface/yarn.lock @@ -11406,10 +11406,10 @@ map-visit@^1.0.0: dependencies: object-visit "^1.0.0" -marked@^13.0.0: - version "13.0.1" - resolved "https://registry.yarnpkg.com/marked/-/marked-13.0.1.tgz#38386e1ebe9c09bb4db72c8ed9759411cb80fd43" - integrity sha512-7kBohS6GrZKvCsNXZyVVXSW7/hGBHe49ng99YPkDCckSUrrG7MSFLCexsRxptzOmyW2eT5dySh4Md1V6my52fA== +marked@^14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/marked/-/marked-14.0.0.tgz#79a1477358a59e0660276f8fec76de2c33f35d83" + integrity sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ== material-colors@^1.2.1: version "1.2.6" From eff2134ec1666620429ad081671173265143fe60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:22:28 +0200 Subject: [PATCH 03/25] Bump posthog-js from 1.141.4 to 1.155.4 in /graylog2-web-interface (#20194) Bumps [posthog-js](https://github.com/PostHog/posthog-js) from 1.141.4 to 1.155.4. - [Release notes](https://github.com/PostHog/posthog-js/releases) - [Changelog](https://github.com/PostHog/posthog-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/PostHog/posthog-js/compare/v1.141.4...v1.155.4) --- updated-dependencies: - dependency-name: posthog-js dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog2-web-interface/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graylog2-web-interface/yarn.lock b/graylog2-web-interface/yarn.lock index a7a157914f6c..53a70f850a35 100644 --- a/graylog2-web-interface/yarn.lock +++ b/graylog2-web-interface/yarn.lock @@ -12808,9 +12808,9 @@ postcss@^8.4.39: source-map-js "^1.2.0" posthog-js@^1.52.0: - version "1.141.4" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.141.4.tgz#c3fcc8bf312c8baaed2b9032cd5fb527a712283b" - integrity sha512-e8rUFEnAR1MB+YqrjWLEmvm0d1X90cebCPNyby6oNX1cp36s/PpxeTx+Up7bArJmRv2N+rT1Kd5sJ7jpXWAonA== + version "1.155.4" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.155.4.tgz#611a63cf95b8fa908b3b50b1043cbdcb4c2a712b" + integrity sha512-suxwAsmZGqMDXJe/RaCKI3PaDEHiuMDDhKcJklgGAg7eDnywieRkr5CoPcOOvnqTDMnuOPETr98jpYBXKUwGFQ== dependencies: fflate "^0.4.8" preact "^10.19.3" From f3e5954e212d95a6795f4d42b698838628194051 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:22:47 +0200 Subject: [PATCH 04/25] Bump stylelint (#20195) Bumps [stylelint](https://github.com/stylelint/stylelint) from 16.7.0 to 16.8.2. - [Release notes](https://github.com/stylelint/stylelint/releases) - [Changelog](https://github.com/stylelint/stylelint/blob/main/CHANGELOG.md) - [Commits](https://github.com/stylelint/stylelint/compare/16.7.0...16.8.2) --- updated-dependencies: - dependency-name: stylelint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../packages/stylelint-config-graylog/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graylog2-web-interface/packages/stylelint-config-graylog/package.json b/graylog2-web-interface/packages/stylelint-config-graylog/package.json index c01732ac2444..4600919038f4 100644 --- a/graylog2-web-interface/packages/stylelint-config-graylog/package.json +++ b/graylog2-web-interface/packages/stylelint-config-graylog/package.json @@ -14,7 +14,7 @@ "license": "SSPL-1.0", "dependencies": { "postcss-styled-syntax": "0.6.4", - "stylelint": "16.7.0", + "stylelint": "16.8.2", "stylelint-config-recommended": "14.0.1", "stylelint-config-standard": "36.0.1", "stylelint-config-styled-components": "0.1.1" From 609eea84eba426c39acaaee0a85cc4a4dc19c0ce Mon Sep 17 00:00:00 2001 From: maxiadlovskii Date: Fri, 16 Aug 2024 09:42:19 +0200 Subject: [PATCH 05/25] Disallows changing field types for immutable index sets (#19936) * Disallows changing field types for immutable index sets * cleaning * cleaning * fix tests * Add hook instead of storage usage --- .../helpers/isIndexFieldTypeChangeAllowed.ts | 2 +- .../indices/hooks/useIndexSetsList.ts | 65 +++++++++++++++++++ .../src/stores/indices/IndexSetsStore.ts | 4 +- .../ChangeFieldTypeModal.test.tsx | 31 +++++++++ .../ChangeFieldType/IndexSetsTable.tsx | 20 +++++- .../hooks/useInitialSelection.tsx | 16 ++++- 6 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 graylog2-web-interface/src/components/indices/hooks/useIndexSetsList.ts diff --git a/graylog2-web-interface/src/components/indices/helpers/isIndexFieldTypeChangeAllowed.ts b/graylog2-web-interface/src/components/indices/helpers/isIndexFieldTypeChangeAllowed.ts index f83de120dde9..7cdb580bf1c6 100644 --- a/graylog2-web-interface/src/components/indices/helpers/isIndexFieldTypeChangeAllowed.ts +++ b/graylog2-web-interface/src/components/indices/helpers/isIndexFieldTypeChangeAllowed.ts @@ -17,7 +17,7 @@ import type { IndexSet } from 'stores/indices/IndexSetsStore'; -const TEMPLATE_TYPES = ['failures', 'events']; +const TEMPLATE_TYPES = ['failures', 'events', 'illuminate_content']; const isIndexFieldTypeChangeAllowed = (indexSet: IndexSet) => !TEMPLATE_TYPES.includes(indexSet?.index_template_type); export default isIndexFieldTypeChangeAllowed; diff --git a/graylog2-web-interface/src/components/indices/hooks/useIndexSetsList.ts b/graylog2-web-interface/src/components/indices/hooks/useIndexSetsList.ts new file mode 100644 index 000000000000..b5a31c3ab4e5 --- /dev/null +++ b/graylog2-web-interface/src/components/indices/hooks/useIndexSetsList.ts @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +import { useQuery } from '@tanstack/react-query'; + +import UserNotification from 'util/UserNotification'; +import fetch from 'logic/rest/FetchProvider'; +import { qualifyUrl } from 'util/URLUtils'; +import ApiRoutes from 'routing/ApiRoutes'; +import type { IndexSet, IndexSetsResponseType, IndexSetsStats } from 'stores/indices/IndexSetsStore'; + +type State = { + indexSetsCount: number, + indexSets: Array, + indexSetStats: IndexSetsStats | null, +} +const getUrl = (stats:boolean) => qualifyUrl(ApiRoutes.IndexSetsApiController.list(stats).url); +const fetchIndexSetsList = (stats:boolean): Promise => fetch('GET', getUrl(stats)).then((response: IndexSetsResponseType) => ({ + indexSetsCount: response.total, + indexSets: response.index_sets, + indexSetStats: response.stats, +})); + +const initialData: State = { indexSets: [], indexSetsCount: null, indexSetStats: null }; + +const useIndexSetsList = (stats: boolean = false) : { + data: State, + refetch: () => void, + isSuccess: boolean, + isInitialLoading: boolean, +} => { + const { data, refetch, isInitialLoading, isSuccess } = useQuery( + ['IndexSetsList', stats], + () => fetchIndexSetsList(stats), + { + onError: (errorThrown) => { + UserNotification.error(`Loading index sets with list failed with status: ${errorThrown}`, + 'Could not load index sets list'); + }, + keepPreviousData: true, + }, + ); + + return ({ + data: data ?? initialData, + refetch, + isSuccess, + isInitialLoading, + }); +}; + +export default useIndexSetsList; diff --git a/graylog2-web-interface/src/stores/indices/IndexSetsStore.ts b/graylog2-web-interface/src/stores/indices/IndexSetsStore.ts index 158389a20d8a..e929320e4015 100644 --- a/graylog2-web-interface/src/stores/indices/IndexSetsStore.ts +++ b/graylog2-web-interface/src/stores/indices/IndexSetsStore.ts @@ -83,11 +83,11 @@ export type IndexSetStats = { size: number, } -type IndexSetsStats = { +export type IndexSetsStats = { [key: string]: IndexSetStats } -type IndexSetsResponseType = { +export type IndexSetsResponseType = { total: number, index_sets: Array, stats: IndexSetsStats, diff --git a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldTypeModal.test.tsx b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldTypeModal.test.tsx index 937af5502b4b..5e22b3f18633 100644 --- a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldTypeModal.test.tsx +++ b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/ChangeFieldTypeModal.test.tsx @@ -18,6 +18,7 @@ import * as React from 'react'; import { render, screen, fireEvent, waitFor } from 'wrappedTestingLibrary'; import selectEvent from 'react-select-event'; +import { MockStore } from 'helpers/mocking'; import asMock from 'helpers/mocking/AsMock'; import useFieldTypeMutation from 'views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypeMutation'; import useFieldTypeUsages from 'views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypeUsages'; @@ -29,6 +30,8 @@ import ChangeFieldTypeModal from 'views/logic/fieldactions/ChangeFieldType/Chang import type { Attributes } from 'stores/PaginationTypes'; import suppressConsole from 'helpers/suppressConsole'; import useFieldTypesForMappings from 'views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypesForMappings'; +import useIndexSetsList from 'components/indices/hooks/useIndexSetsList'; +import type { IndexSet } from 'stores/indices/IndexSetsStore'; const onCloseMock = jest.fn(); const renderChangeFieldTypeModal = ({ @@ -116,6 +119,20 @@ jest.mock('views/logic/fieldactions/ChangeFieldType/hooks/useFieldTypeMutation', jest.mock('components/common/EntityDataTable/hooks/useUserLayoutPreferences'); +jest.mock('components/indices/hooks/useIndexSetsList'); + +jest.mock('stores/indices/IndexSetsStore', () => ({ + IndexSetsActions: { + list: jest.fn(), + }, + IndexSetsStore: MockStore(['getInitialState', () => ({ + indexSets: [ + { id: 'id-1', title: 'Index Title 1' }, + { id: 'id-2', title: 'Index Title 2' }, + ], + })]), +})); + describe('ChangeFieldTypeModal', () => { const putFieldTypeMutationMock = jest.fn(() => Promise.resolve()); @@ -133,6 +150,20 @@ describe('ChangeFieldTypeModal', () => { }, }); + asMock(useIndexSetsList).mockReturnValue({ + data: { + indexSets: [ + { id: 'id-1', title: 'Index Title 1' }, + { id: 'id-2', title: 'Index Title 2' }, + ] as Array, + indexSetsCount: 2, + indexSetStats: null, + }, + isSuccess: true, + isInitialLoading: false, + refetch: () => {}, + }); + asMock(useFieldTypeMutation).mockReturnValue({ isLoading: false, putFieldTypeMutation: putFieldTypeMutationMock }); asMock(useFieldTypeUsages).mockReturnValue(paginatedFieldUsage); diff --git a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/IndexSetsTable.tsx b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/IndexSetsTable.tsx index 8cbfbbd09a96..1074f498983f 100644 --- a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/IndexSetsTable.tsx +++ b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/IndexSetsTable.tsx @@ -32,6 +32,9 @@ import type { FieldTypeUsage, FieldTypes } from 'views/logic/fieldactions/Change import useColumnRenderers from 'views/logic/fieldactions/ChangeFieldType/hooks/useColumnRenderers'; import BulkActionsDropdown from 'components/common/EntityDataTable/BulkActionsDropdown'; import useCurrentStream from 'views/logic/fieldactions/ChangeFieldType/hooks/useCurrentStream'; +import isIndexFieldTypeChangeAllowed from 'components/indices/helpers/isIndexFieldTypeChangeAllowed'; +import useIndexSetsList from 'components/indices/hooks/useIndexSetsList'; +import type { IndexSet } from 'stores/indices/IndexSetsStore'; const Container = styled.div` margin-top: 20px; @@ -44,8 +47,16 @@ type Props = { initialSelection: Array } +const mapper = (indexSets: Array): Record => { + if (!indexSets) return null; + + return Object.fromEntries(indexSets.map((indexSet) => ([indexSet.id, indexSet]))); +}; + const IndexSetsTable = ({ field, setIndexSetSelection, fieldTypes, initialSelection }: Props) => { const [activePage, setActivePage] = useState(1); + const { data, isSuccess } = useIndexSetsList(); + const mappedIndexSets = useMemo(() => mapper(data?.indexSets), [data?.indexSets]); const { layoutConfig, isInitialLoading: isLoadingLayoutPreferences } = useTableLayout({ entityTableId: ENTITY_TABLE_ID, @@ -96,7 +107,13 @@ const IndexSetsTable = ({ field, setIndexSetSelection, fieldTypes, initialSelect setIndexSetSelection(newSelection); }, [setIndexSetSelection]); - if (isLoadingLayoutPreferences || isLoading) { + const isEntitySelectable = useCallback((entity) => { + const indexSetId = entity.id; + + return isIndexFieldTypeChangeAllowed(mappedIndexSets[indexSetId]); + }, [mappedIndexSets]); + + if (isLoadingLayoutPreferences || isLoading || !isSuccess) { return ; } @@ -119,6 +136,7 @@ const IndexSetsTable = ({ field, setIndexSetSelection, fieldTypes, initialSelect onChangeSelection, initialSelection, actions: , + isEntitySelectable, }} columnDefinitions={attributes} columnRenderers={columnRenderers} diff --git a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useInitialSelection.tsx b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useInitialSelection.tsx index 38746beeef1c..5cc74f892d8a 100644 --- a/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useInitialSelection.tsx +++ b/graylog2-web-interface/src/views/logic/fieldactions/ChangeFieldType/hooks/useInitialSelection.tsx @@ -20,19 +20,29 @@ import { useStore } from 'stores/connect'; import type { Stream } from 'views/stores/StreamsStore'; import { StreamsStore } from 'views/stores/StreamsStore'; import useCurrentStream from 'views/logic/fieldactions/ChangeFieldType/hooks/useCurrentStream'; +import type { IndexSet, IndexSetsStoreState } from 'stores/indices/IndexSetsStore'; +import { IndexSetsStore } from 'stores/indices/IndexSetsStore'; +import isIndexFieldTypeChangeAllowed from 'components/indices/helpers/isIndexFieldTypeChangeAllowed'; const streamsMapper = ({ streams }) => streams.map((stream: Stream) => ({ indexSet: stream.index_set_id, id: stream.id })); +const indexSetsStoreMapper = ({ indexSets }: IndexSetsStoreState): Record => { + if (!indexSets) return null; + + return Object.fromEntries(indexSets.map((indexSet) => ([indexSet.id, indexSet]))); +}; + const useInitialSelection = () => { const currentStreams = useCurrentStream(); + const indexSets = useStore(IndexSetsStore, indexSetsStoreMapper); const availableStreams: Array<{ indexSet: string, id: string }> = useStore(StreamsStore, streamsMapper); return useMemo(() => { const currentStreamSet = new Set(currentStreams); - const filterFn = currentStreamSet.size > 0 ? ({ id }) => currentStreamSet.has(id) : () => true; + const filterFn = currentStreamSet.size > 0 ? ({ id, indexSet }) => currentStreamSet.has(id) && isIndexFieldTypeChangeAllowed(indexSets[indexSet]) : () => true; - return availableStreams.filter(filterFn).map(({ indexSet }) => indexSet); - }, [availableStreams, currentStreams]); + return indexSets ? availableStreams.filter(filterFn).map(({ indexSet }) => indexSet) : []; + }, [availableStreams, currentStreams, indexSets]); }; export default useInitialSelection; From 61d46caf5756ef5859211fbdfc3fd61905ce8fff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:10:01 +0200 Subject: [PATCH 06/25] Bump junit-jupiter.version from 5.10.3 to 5.11.0 (#20183) * Bump junit-jupiter.version from 5.10.3 to 5.11.0 Bumps `junit-jupiter.version` from 5.10.3 to 5.11.0. Updates `org.junit:junit-bom` from 5.10.3 to 5.11.0 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.3...r5.11.0) Updates `org.junit.jupiter:junit-jupiter` from 5.10.3 to 5.11.0 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.3...r5.11.0) Updates `org.junit.vintage:junit-vintage-engine` from 5.10.3 to 5.11.0 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.10.3...r5.11.0) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.junit.jupiter:junit-jupiter dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: org.junit.vintage:junit-vintage-engine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Adjust container matrix code to JUnit 5.11 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bernd Ahlers --- .../engine/descriptor/ContainerMatrixEngineDescriptor.java | 3 +-- .../engine/descriptor/ContainerMatrixExtensionContext.java | 7 +++---- pom.xml | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixEngineDescriptor.java b/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixEngineDescriptor.java index 294a0ea60447..2aff7d56be66 100644 --- a/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixEngineDescriptor.java +++ b/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixEngineDescriptor.java @@ -58,9 +58,8 @@ public JupiterEngineExecutionContext prepare(JupiterEngineExecutionContext conte MutableExtensionRegistry extensionRegistry = MutableExtensionRegistry.createRegistryWithDefaultExtensions( context.getConfiguration()); EngineExecutionListener executionListener = context.getExecutionListener(); - ExecutableInvoker executableInvoker = new DefaultExecutableInvoker(context); ExtensionContext extensionContext = new ContainerMatrixExtensionContext(executionListener, this, - context.getConfiguration(), executableInvoker); + context.getConfiguration()); // @formatter:off return context.extend() diff --git a/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixExtensionContext.java b/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixExtensionContext.java index 15d081deee46..cb8affe52ed9 100644 --- a/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixExtensionContext.java +++ b/graylog2-server/src/test/java/org/junit/jupiter/engine/descriptor/ContainerMatrixExtensionContext.java @@ -17,7 +17,7 @@ package org.junit.jupiter.engine.descriptor; import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.extension.ExecutableInvoker; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.TestInstances; import org.junit.jupiter.engine.config.JupiterConfiguration; import org.junit.platform.engine.EngineExecutionListener; @@ -31,10 +31,9 @@ public class ContainerMatrixExtensionContext extends AbstractExtensionContext3.16.1 1.5 4.13.2 - 5.10.3 + 5.11.0 5.12.0 5.5.0 1.19.0 From 995e0c3883c0795227ae6b41669d6ecd24c6ff8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:32:38 +0200 Subject: [PATCH 07/25] Bump byte-buddy.version from 1.14.18 to 1.14.19 (#20192) Bumps `byte-buddy.version` from 1.14.18 to 1.14.19. Updates `net.bytebuddy:byte-buddy` from 1.14.18 to 1.14.19 - [Release notes](https://github.com/raphw/byte-buddy/releases) - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) - [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.14.18...byte-buddy-1.14.19) Updates `net.bytebuddy:byte-buddy-agent` from 1.14.18 to 1.14.19 - [Release notes](https://github.com/raphw/byte-buddy/releases) - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) - [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.14.18...byte-buddy-1.14.19) --- updated-dependencies: - dependency-name: net.bytebuddy:byte-buddy dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: net.bytebuddy:byte-buddy-agent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31c40d0dc4c3..a80a866a93f3 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 2.6.0 2.2.0 1.78.1 - 1.14.18 + 1.14.19 3.1.8 0.0.1.10 4.8.174 From ee9a81d48e098428862a6ae1afbb41c82a8f8fea Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Fri, 16 Aug 2024 13:27:48 +0200 Subject: [PATCH 08/25] Handle closed indices in remote reindex migration (#20081) * Handle closed indices in remote reindex migration * selectAll shouldn't select the closed indices * added changelog * Remove support for empty indices list in remote reindex migration (not used anywhere) * Reopen closed indices for remote reindex migration * updated changelog * fix setting authorization header only for non-null parameters * remove check for message from closed index --------- Co-authored-by: Mohamed OULD HOCINE <106236152+gally47@users.noreply.github.com> Co-authored-by: Mohamed Ould Hocine Co-authored-by: Matthias Oesterheld <33032967+moesterheld@users.noreply.github.com> Co-authored-by: Matthias Oesterheld --- changelog/unreleased/pr-20081.toml | 5 + .../datanode/rest/IndexStateController.java | 121 ++++++++++++++++++ .../OpensearchConnectionCheckController.java | 20 ++- .../graylog/datanode/rest/RestBindings.java | 1 + .../datanode/RemoteReindexingMigrationIT.java | 8 +- .../AggregatedConnectionResponse.java | 4 +- .../opensearch2/ConnectionCheckIndex.java | 21 +++ .../opensearch2/ConnectionCheckResponse.java | 4 +- .../DatanodeRemoteIndexStateResource.java | 35 +++++ .../storage/opensearch2/IndexState.java | 21 +++ .../opensearch2/IndexStateChangeRequest.java | 20 +++ .../opensearch2/IndexStateGetRequest.java | 20 +++ .../RemoteReindexingMigrationAdapterOS2.java | 53 +++++++- .../AggregatedConnectionResponseTest.java | 24 ++-- .../indexer/migration/RemoteIndex.java | 2 +- .../datanodes/DatanodeRestApiProxy.java | 2 +- .../remoteReindexing/MigrateExistingData.tsx | 7 +- 17 files changed, 339 insertions(+), 29 deletions(-) create mode 100644 changelog/unreleased/pr-20081.toml create mode 100644 data-node/src/main/java/org/graylog/datanode/rest/IndexStateController.java create mode 100644 graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckIndex.java create mode 100644 graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/DatanodeRemoteIndexStateResource.java create mode 100644 graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexState.java create mode 100644 graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateChangeRequest.java create mode 100644 graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateGetRequest.java diff --git a/changelog/unreleased/pr-20081.toml b/changelog/unreleased/pr-20081.toml new file mode 100644 index 000000000000..5f830d20e855 --- /dev/null +++ b/changelog/unreleased/pr-20081.toml @@ -0,0 +1,5 @@ +type = "f" +message = "Add support for remote-reindex migration of closed indices" + +pulls = ["20081"] +issues = ["20068"] diff --git a/data-node/src/main/java/org/graylog/datanode/rest/IndexStateController.java b/data-node/src/main/java/org/graylog/datanode/rest/IndexStateController.java new file mode 100644 index 000000000000..9ccb4dc2957f --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/rest/IndexStateController.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.rest; + +import jakarta.inject.Inject; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import org.graylog.datanode.configuration.DatanodeTrustManagerProvider; +import org.graylog.storage.opensearch2.IndexState; +import org.graylog.storage.opensearch2.IndexStateChangeRequest; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.Locale; +import java.util.Objects; + +import static org.graylog.datanode.rest.OpensearchConnectionCheckController.CONNECT_TIMEOUT; +import static org.graylog.datanode.rest.OpensearchConnectionCheckController.READ_TIMEOUT; +import static org.graylog.datanode.rest.OpensearchConnectionCheckController.WRITE_TIMEOUT; + +@Path("/index-state") +@Produces(MediaType.APPLICATION_JSON) +public class IndexStateController { + + private final DatanodeTrustManagerProvider datanodeTrustManagerProvider; + private final OkHttpClient httpClient; + + @Inject + public IndexStateController(DatanodeTrustManagerProvider datanodeTrustManagerProvider) { + this.datanodeTrustManagerProvider = datanodeTrustManagerProvider; + this.httpClient = new OkHttpClient.Builder() + .retryOnConnectionFailure(true) + .connectTimeout(CONNECT_TIMEOUT) + .writeTimeout(WRITE_TIMEOUT) + .readTimeout(READ_TIMEOUT) + .build(); + } + + @POST + @Path("/get") + public IndexState get(IndexStateChangeRequest indexStateChangeRequest) { + final String host = indexStateChangeRequest.host().endsWith("/") ? indexStateChangeRequest.host() : indexStateChangeRequest.host() + "/"; + final Request.Builder request = new Request.Builder() + .url(host + "_cat/indices/" + indexStateChangeRequest.indexName() + "/?h=status"); + if (Objects.nonNull(indexStateChangeRequest.username()) && Objects.nonNull(indexStateChangeRequest.password())) { + request.header("Authorization", Credentials.basic(indexStateChangeRequest.username(), indexStateChangeRequest.password())); + } + try (var response = getClient().newCall(request.build()).execute()) { + if (response.isSuccessful() && response.body() != null) { + final String state = response.body().string().trim().toUpperCase(Locale.ROOT); + return IndexState.valueOf(state); + } else { + throw new RuntimeException("Failed to detect open/close index status " + indexStateChangeRequest.indexName() + ". Code: " + response.code() + "; message=" + response.message()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to open/close index" + indexStateChangeRequest.indexName(), e); + } + } + + @POST + @Path("/set") + public IndexState change(IndexStateChangeRequest indexStateChangeRequest) { + return performAction(indexStateChangeRequest); + } + + private IndexState performAction(IndexStateChangeRequest indexStateChangeRequest) { + final String host = indexStateChangeRequest.host().endsWith("/") ? indexStateChangeRequest.host() : indexStateChangeRequest.host() + "/"; + final Request.Builder request = new Request.Builder() + .post(RequestBody.create("", okhttp3.MediaType.parse(MediaType.APPLICATION_JSON))) + .url(host + indexStateChangeRequest.indexName() + "/" + (indexStateChangeRequest.action() == IndexState.OPEN ? "_open" : "_close")); + if (Objects.nonNull(indexStateChangeRequest.username()) && Objects.nonNull(indexStateChangeRequest.password())) { + request.header("Authorization", Credentials.basic(indexStateChangeRequest.username(), indexStateChangeRequest.password())); + } + try (var response = getClient().newCall(request.build()).execute()) { + if (response.isSuccessful()) { + return indexStateChangeRequest.action(); + } else { + throw new RuntimeException("Failed to open/close index " + indexStateChangeRequest.indexName() + ". Code: " + response.code() + "; message=" + response.message()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to open/close index" + indexStateChangeRequest.indexName(), e); + } + } + + private OkHttpClient getClient() { + try { + final SSLContext ctx = SSLContext.getInstance("TLS"); + final X509TrustManager trustManager = datanodeTrustManagerProvider.get(); + ctx.init(null, new TrustManager[]{trustManager}, new SecureRandom()); + return httpClient.newBuilder().sslSocketFactory(ctx.getSocketFactory(), trustManager).build(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException(e); + + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/rest/OpensearchConnectionCheckController.java b/data-node/src/main/java/org/graylog/datanode/rest/OpensearchConnectionCheckController.java index 96170bb78226..5a11826a1505 100644 --- a/data-node/src/main/java/org/graylog/datanode/rest/OpensearchConnectionCheckController.java +++ b/data-node/src/main/java/org/graylog/datanode/rest/OpensearchConnectionCheckController.java @@ -26,6 +26,7 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import org.graylog.datanode.configuration.DatanodeTrustManagerProvider; +import org.graylog.storage.opensearch2.ConnectionCheckIndex; import org.graylog.storage.opensearch2.ConnectionCheckRequest; import org.graylog.storage.opensearch2.ConnectionCheckResponse; import org.graylog2.security.TrustAllX509TrustManager; @@ -42,6 +43,7 @@ import java.security.SecureRandom; import java.security.cert.X509Certificate; import java.time.Duration; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -74,19 +76,24 @@ public ConnectionCheckResponse status(ConnectionCheckRequest request) { final List unknownCertificates = new LinkedList<>(); try { unknownCertificates.addAll(extractUnknownCertificates(request.host())); - final List indices = getAllIndicesFrom(request.host(), request.username(), request.password(), request.trustUnknownCerts()); + final List indices = getAllIndicesFrom(request.host(), request.username(), request.password(), request.trustUnknownCerts()); return ConnectionCheckResponse.success(indices, unknownCertificates); } catch (Exception e) { return ConnectionCheckResponse.error(e, unknownCertificates); } } - List getAllIndicesFrom(final String host, final String username, final String password, boolean trustUnknownCerts) { - var url = (host.endsWith("/") ? host : host + "/") + "_cat/indices?h=index"; + List getAllIndicesFrom(final String host, final String username, final String password, boolean trustUnknownCerts) { + var url = (host.endsWith("/") ? host : host + "/") + "_cat/indices?h=index,status"; try (var response = getClient(trustUnknownCerts).newCall(new Request.Builder().url(url).header("Authorization", Credentials.basic(username, password)).build()).execute()) { if (response.isSuccessful() && response.body() != null) { // filtering all indices that start with "." as they indicate a system index - we don't want to reindex those - return new BufferedReader(new StringReader(response.body().string())).lines().filter(i -> !i.startsWith(".")).sorted().toList(); + return new BufferedReader(new StringReader(response.body().string())) + .lines() + .filter(i -> !i.startsWith(".")) + .map(this::parseIndexLine) + .sorted(Comparator.comparing(ConnectionCheckIndex::name, Comparator.naturalOrder())) + .toList(); } else { String message = String.format(Locale.ROOT, "Could not read list of indices from %s. Code=%d, message=%s", host, response.code(), response.message()); throw new RuntimeException(message); @@ -96,6 +103,11 @@ List getAllIndicesFrom(final String host, final String username, final S } } + private ConnectionCheckIndex parseIndexLine(String line) { + final String[] parts = line.split("\\s+"); + return new ConnectionCheckIndex(parts[0], parts[1].contains("close")); + } + private OkHttpClient getClient(boolean trustUnknownCerts) { try { diff --git a/data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java b/data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java index e949bb8dc8e5..c7da3594af20 100644 --- a/data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java +++ b/data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java @@ -26,6 +26,7 @@ protected void configure() { addSystemRestResource(ManagementController.class); addSystemRestResource(IndicesDirectoryController.class); addSystemRestResource(OpensearchConnectionCheckController.class); + addSystemRestResource(IndexStateController.class); addSystemRestResource(CertificatesController.class); } } diff --git a/full-backend-tests/src/test/java/org/graylog/datanode/RemoteReindexingMigrationIT.java b/full-backend-tests/src/test/java/org/graylog/datanode/RemoteReindexingMigrationIT.java index 9b799b12cbf9..46d1261c10f0 100644 --- a/full-backend-tests/src/test/java/org/graylog/datanode/RemoteReindexingMigrationIT.java +++ b/full-backend-tests/src/test/java/org/graylog/datanode/RemoteReindexingMigrationIT.java @@ -27,6 +27,7 @@ import org.assertj.core.api.Assertions; import org.graylog.shaded.opensearch2.org.opensearch.action.index.IndexRequest; import org.graylog.shaded.opensearch2.org.opensearch.action.index.IndexResponse; +import org.graylog.shaded.opensearch2.org.opensearch.client.indices.CloseIndexRequest; import org.graylog.shaded.opensearch2.org.opensearch.client.indices.CreateIndexRequest; import org.graylog.shaded.opensearch2.org.opensearch.client.indices.CreateIndexResponse; import org.graylog.storage.opensearch2.testing.OpenSearchInstance; @@ -78,6 +79,8 @@ void testRemoteAsyncReindexing() throws ExecutionException, RetryException { final String messageContent = ingestRandomMessage(indexName); final String messageContent2 = ingestRandomMessage(indexName2); + closeIndex(indexName); + // flush the newly created document openSearchInstance.client().refreshNode(); @@ -99,11 +102,14 @@ void testRemoteAsyncReindexing() throws ExecutionException, RetryException { final String status = response.extract().body().jsonPath().get("status"); Assertions.assertThat(status).isEqualTo("FINISHED"); - Assertions.assertThat(waitForMessage(indexName, messageContent)).containsEntry("message", messageContent); Assertions.assertThat(waitForMessage(indexName2, messageContent2)).containsEntry("message", messageContent2); } + private void closeIndex(String indexName) { + openSearchInstance.openSearchClient().execute((restHighLevelClient, requestOptions) -> restHighLevelClient.indices().close(new CloseIndexRequest(indexName), requestOptions)); + } + /** * @return name of the newly created index */ diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/AggregatedConnectionResponse.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/AggregatedConnectionResponse.java index 241db2aa45ea..435441c63c34 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/AggregatedConnectionResponse.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/AggregatedConnectionResponse.java @@ -39,11 +39,11 @@ public record AggregatedConnectionResponse(Map responses) { @Nonnull - public List indices() { + public List indices() { return responses.values().stream() .filter(v -> Objects.nonNull(v.indices())) .flatMap(v -> v.indices().stream()) - .sorted(Comparator.naturalOrder()) + .sorted(Comparator.comparing(ConnectionCheckIndex::name, Comparator.naturalOrder())) .distinct() .collect(Collectors.toList()); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckIndex.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckIndex.java new file mode 100644 index 000000000000..9ec0bbba68ee --- /dev/null +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckIndex.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.storage.opensearch2; + +public record ConnectionCheckIndex(String name, boolean closed) { + +} diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckResponse.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckResponse.java index 64e4429c75cd..6e4ddf0e885c 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckResponse.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/ConnectionCheckResponse.java @@ -29,9 +29,9 @@ import java.util.stream.Collectors; @JsonInclude(JsonInclude.Include.NON_EMPTY) -public record ConnectionCheckResponse(List indices, List certificates, +public record ConnectionCheckResponse(List indices, List certificates, String error) { - public static ConnectionCheckResponse success(List indices, List certificates) { + public static ConnectionCheckResponse success(List indices, List certificates) { return new ConnectionCheckResponse(indices, encodeCerts(certificates), null); } diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/DatanodeRemoteIndexStateResource.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/DatanodeRemoteIndexStateResource.java new file mode 100644 index 000000000000..6eb65948c348 --- /dev/null +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/DatanodeRemoteIndexStateResource.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.storage.opensearch2; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import retrofit2.Call; +import retrofit2.http.Body; +import retrofit2.http.GET; +import retrofit2.http.POST; + +public interface DatanodeRemoteIndexStateResource { + + @POST("index-state/get") + Call readState(@Body @Valid @NotNull IndexStateGetRequest request); + + @POST("index-state/set") + Call changeState(@Body @Valid @NotNull IndexStateChangeRequest request); + + +} diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexState.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexState.java new file mode 100644 index 000000000000..cf7ebe316e6c --- /dev/null +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexState.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.storage.opensearch2; + +public enum IndexState { + OPEN, CLOSE +} diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateChangeRequest.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateChangeRequest.java new file mode 100644 index 000000000000..aabd84694be4 --- /dev/null +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateChangeRequest.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.storage.opensearch2; + +public record IndexStateChangeRequest(String indexName, IndexState action, String host, String username, String password) { +} diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateGetRequest.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateGetRequest.java new file mode 100644 index 000000000000..1319901e5932 --- /dev/null +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/IndexStateGetRequest.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.storage.opensearch2; + +public record IndexStateGetRequest(String indexName, String host, String username, String password) { +} diff --git a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java index 1c748b3dfc70..244ea7cccb90 100644 --- a/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java +++ b/graylog-storage-opensearch2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java @@ -38,6 +38,7 @@ import org.graylog.shaded.opensearch2.org.opensearch.client.RequestOptions; import org.graylog.shaded.opensearch2.org.opensearch.client.Response; import org.graylog.shaded.opensearch2.org.opensearch.client.ResponseException; +import org.graylog.shaded.opensearch2.org.opensearch.client.indices.CloseIndexRequest; import org.graylog.shaded.opensearch2.org.opensearch.client.tasks.TaskSubmissionResponse; import org.graylog.shaded.opensearch2.org.opensearch.cluster.health.ClusterHealthStatus; import org.graylog.shaded.opensearch2.org.opensearch.common.xcontent.json.JsonXContent; @@ -160,8 +161,7 @@ private Boolean isIndexSetCurrentlyMigrated(IndexSet indexSet, RemoteReindexMigr @Override public String start(RemoteReindexRequest request) { final AggregatedConnectionResponse response = getAllIndicesFrom(request.uri(), request.username(), request.password(), request.trustUnknownCerts()); - final List indices = isAllIndices(request.indices()) ? response.indices() : request.indices(); - final MigrationConfiguration migration = reindexMigrationService.saveMigration(MigrationConfiguration.forIndices(indices, response.certificates())); + final MigrationConfiguration migration = reindexMigrationService.saveMigration(MigrationConfiguration.forIndices(request.indices(), response.certificates())); doStartMigration(migration, request); return migration.id(); @@ -282,7 +282,7 @@ public IndexerConnectionCheckResult checkConnection(@Nonnull URI remoteHost, @Nu reindexAllowlist.validate(); final AggregatedConnectionResponse results = getAllIndicesFrom(remoteHost, username, password, trustUnknownCerts); final List indices = results.indices().stream() - .map(i -> new RemoteIndex(i, indexSetRegistry.isManagedIndex(i))) + .map(i -> new RemoteIndex(i.name(), indexSetRegistry.isManagedIndex(i.name()), i.closed())) .distinct() .toList(); if (results.error() != null && !results.error().isEmpty()) { @@ -384,10 +384,6 @@ private static String uriToString(URI uri) { } } - boolean isAllIndices(final List indices) { - return indices == null || indices.isEmpty() || (indices.size() == 1 && "*".equals(indices.get(0))); - } - private void startAsyncTasks(MigrationConfiguration migration, RemoteReindexRequest request) { final int threadsCount = Math.max(1, Math.min(request.threadsCount(), migration.indices().size())); @@ -403,7 +399,9 @@ private void startAsyncTasks(MigrationConfiguration migration, RemoteReindexRequ private void executeReindexAsync(MigrationConfiguration migration, URI uri, String username, String password, IndexMigrationConfiguration index) { final String indexName = index.indexName(); + try (XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint()) { + final boolean closeAfterMigration = openIndexIfNeeded(migration, uri, username, password, indexName); final BytesReference query = BytesReference.bytes(matchAllQuery().toXContent(builder, ToXContent.EMPTY_PARAMS)); logInfo(migration, "Executing async reindex for " + indexName); final TaskSubmissionResponse task = client.execute((c, requestOptions) -> { @@ -414,12 +412,53 @@ private void executeReindexAsync(MigrationConfiguration migration, URI uri, Stri }); reindexMigrationService.assignTask(migration.id(), indexName, task.getTask()); waitForTaskCompleted(migration, indexName, task.getTask()); + if(closeAfterMigration) { + closeMigratedIndex(migration, uri, username, password, indexName); + } } catch (Exception e) { final String message = "Could not reindex index: " + indexName + " - " + e.getMessage(); logError(migration, message, e); } } + private void closeMigratedIndex(MigrationConfiguration migration, URI uri, String username, String password, String indexName) { + closeRemoteIndex(uri, username, password, indexName); + client.execute((restHighLevelClient, requestOptions) -> restHighLevelClient.indices().close(new CloseIndexRequest(indexName), requestOptions)); + logInfo(migration, "Restoring original index state, both source and target index " + indexName + " closed after migration"); + } + + private boolean openIndexIfNeeded(MigrationConfiguration migration, URI uri, String username, String password, String indexName) { + final IndexState indexState = getIndexState(uri, username, password, indexName); + + boolean closeAfterMigration = false; + + if(indexState == IndexState.CLOSE) { + logInfo(migration, "Source index " + indexName + " is closed, reopening for the migration"); + closeAfterMigration = true; + openRemoteIndex(uri, username, password, indexName); + } + return closeAfterMigration; + } + + private void openRemoteIndex(URI uri, String username, String password, String indexName) { + datanodeRestApiProxy.remoteInterface(DatanodeResolver.ANY_NODE_KEYWORD, DatanodeRemoteIndexStateResource.class, datanodeRemoteIndexStateResource -> datanodeRemoteIndexStateResource.changeState(new IndexStateChangeRequest( + indexName, IndexState.OPEN, uri.toString(), username, password + ))); + } + + private void closeRemoteIndex(URI uri, String username, String password, String indexName) { + datanodeRestApiProxy.remoteInterface(DatanodeResolver.ANY_NODE_KEYWORD, DatanodeRemoteIndexStateResource.class, datanodeRemoteIndexStateResource -> datanodeRemoteIndexStateResource.changeState(new IndexStateChangeRequest( + indexName, IndexState.CLOSE, uri.toString(), username, password + ))); + } + + private IndexState getIndexState(URI uri, String username, String password, String indexName) { + final IndexState indexState = datanodeRestApiProxy.remoteInterface(DatanodeResolver.ANY_NODE_KEYWORD, DatanodeRemoteIndexStateResource.class, resource -> resource.readState(new IndexStateGetRequest( + indexName, uri.toString(), username, password + ))).values().iterator().next(); + return indexState; + } + private void logInfo(MigrationConfiguration migration, String message) { LOG.info(message); diff --git a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/AggregatedConnectionResponseTest.java b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/AggregatedConnectionResponseTest.java index 9a4bb11b8adb..cdd12bfad3d2 100644 --- a/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/AggregatedConnectionResponseTest.java +++ b/graylog-storage-opensearch2/src/test/java/org/graylog/storage/opensearch2/AggregatedConnectionResponseTest.java @@ -36,13 +36,17 @@ class AggregatedConnectionResponseTest { @Test void testAggregateIndices() { final AggregatedConnectionResponse aggregatedResponse = new AggregatedConnectionResponse(Map.of( - "node-one", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.emptyList(), null), - "node-two", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.emptyList(), null), - "node-three", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.emptyList(), null) + "node-one", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.emptyList(), null), + "node-two", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.emptyList(), null), + "node-three", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.emptyList(), null) )); Assertions.assertThat(aggregatedResponse.indices()) .hasSize(3) - .contains("graylog_0", "graylog_1", "graylog_2"); + .contains(i("graylog_0"), i("graylog_1"), i("graylog_2")); + } + + private ConnectionCheckIndex i(String indexName) { + return new ConnectionCheckIndex(indexName, false); } @Test @@ -75,9 +79,9 @@ void testAggregateCertificates() throws Exception { final AggregatedConnectionResponse aggregatedResponse = new AggregatedConnectionResponse(Map.of( - "node-one", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), null), - "node-two", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), null), - "node-three", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), null) + "node-one", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), null), + "node-two", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), null), + "node-three", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), null) )); Assertions.assertThat(aggregatedResponse.certificates()) @@ -94,9 +98,9 @@ void testAggregateCertificatesWithError() throws Exception { final AggregatedConnectionResponse aggregatedResponse = new AggregatedConnectionResponse(Map.of( - "node-one", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), "Unknown cert"), - "node-two", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), "Unknown cert"), - "node-three", new ConnectionCheckResponse(List.of("graylog_0", "graylog_1", "graylog_2"), Collections.singletonList(pemCert), "Unknown cert") + "node-one", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), "Unknown cert"), + "node-two", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), "Unknown cert"), + "node-three", new ConnectionCheckResponse(List.of(i("graylog_0"), i("graylog_1"), i("graylog_2")), Collections.singletonList(pemCert), "Unknown cert") )); Assertions.assertThat(aggregatedResponse.error()) diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java b/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java index a8d6f26ead99..68bd27fed517 100644 --- a/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java +++ b/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java @@ -16,5 +16,5 @@ */ package org.graylog2.indexer.migration; -public record RemoteIndex(String name, boolean managed) { +public record RemoteIndex(String name, boolean managed, boolean closed) { } diff --git a/graylog2-server/src/main/java/org/graylog2/rest/resources/datanodes/DatanodeRestApiProxy.java b/graylog2-server/src/main/java/org/graylog2/rest/resources/datanodes/DatanodeRestApiProxy.java index 7cd1d43e5e74..d1478f6e35b7 100644 --- a/graylog2-server/src/main/java/org/graylog2/rest/resources/datanodes/DatanodeRestApiProxy.java +++ b/graylog2-server/src/main/java/org/graylog2/rest/resources/datanodes/DatanodeRestApiProxy.java @@ -166,7 +166,7 @@ public Map .build(); try { final retrofit2.Response response = function.apply(retrofit.create(interfaceClass)).execute(); - if (response.isSuccessful() && response.body() != null) { + if (response.isSuccessful() && response.body() != null) { // TODO: this causes exceptions when the remote if returns no body but ok state! return response.body(); } else { throw new IllegalStateException("Failed to trigger datanode request. Code: " + response.code() + ", message: " + response.message()); diff --git a/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx b/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx index a9d6d709e318..bd947ea6e37c 100644 --- a/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx +++ b/graylog2-web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx @@ -43,6 +43,7 @@ const SearchContainer = styled.div` type RemoteIndex = { name: string, managed: boolean, + closed: boolean, } type RemoteReindexCheckConnection = { @@ -228,10 +229,14 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat label={( <> {index.name} - {!index.managed && ( + {!index.managed && !index.closed && ( )} + {index.closed && ( + + )} )} disabled={isLoading} From 53ff861e5c72aec0a25cfe68ef108214dcf83535 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Fri, 16 Aug 2024 14:18:08 +0200 Subject: [PATCH 09/25] Bump package.json version to 6.1.0-alpha.7 --- graylog2-web-interface/manifests/package.json | 2 +- graylog2-web-interface/package.json | 2 +- graylog2-web-interface/packages/graylog-web-plugin/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graylog2-web-interface/manifests/package.json b/graylog2-web-interface/manifests/package.json index 5ee9df35fe99..c7f624d4fe8c 100644 --- a/graylog2-web-interface/manifests/package.json +++ b/graylog2-web-interface/manifests/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-manifests", - "version": "6.1.0-SNAPSHOT", + "version": "6.1.0-alpha.7", "description": "Manifests needed to reuse vendor & shared bundles in graylog web interface plugins", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/graylog2-web-interface/package.json b/graylog2-web-interface/package.json index 3d0ca6778b29..121f94a114f2 100644 --- a/graylog2-web-interface/package.json +++ b/graylog2-web-interface/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-interface", - "version": "6.1.0-SNAPSHOT", + "version": "6.1.0-alpha.7", "description": "Graylog Web Interface", "author": "torch", "license": "SSPL-1.0", diff --git a/graylog2-web-interface/packages/graylog-web-plugin/package.json b/graylog2-web-interface/packages/graylog-web-plugin/package.json index a8b545c07f30..cac0968b129a 100644 --- a/graylog2-web-interface/packages/graylog-web-plugin/package.json +++ b/graylog2-web-interface/packages/graylog-web-plugin/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-plugin", - "version": "6.1.0-SNAPSHOT", + "version": "6.1.0-alpha.7", "description": "Helper code for streamlining Graylog web interface plugin development", "main": "index.js", "repository": { From a259c1dfebda23d601abe1e3b78fa1d8992598ea Mon Sep 17 00:00:00 2001 From: Jenkins Date: Fri, 16 Aug 2024 14:18:11 +0200 Subject: [PATCH 10/25] [graylog2-server] prepare release 6.1.0-alpha.7 --- data-node/pom.xml | 2 +- distribution/pom.xml | 2 +- full-backend-tests/pom.xml | 2 +- graylog-plugin-archetype/pom.xml | 2 +- graylog-plugin-parent/graylog-plugin-web-parent/pom.xml | 2 +- graylog-plugin-parent/pom.xml | 2 +- graylog-project-parent/pom.xml | 2 +- graylog-storage-elasticsearch7/pom.xml | 2 +- graylog-storage-opensearch2/pom.xml | 2 +- graylog2-server/pom.xml | 2 +- pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data-node/pom.xml b/data-node/pom.xml index 3afbe102faab..f9c9165c1bde 100644 --- a/data-node/pom.xml +++ b/data-node/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-project-parent diff --git a/distribution/pom.xml b/distribution/pom.xml index 3bd1d10c1b66..9ec854616a0b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ graylog-parent org.graylog - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 4.0.0 diff --git a/full-backend-tests/pom.xml b/full-backend-tests/pom.xml index 047b1ffd2df6..23363f39c973 100644 --- a/full-backend-tests/pom.xml +++ b/full-backend-tests/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-project-parent diff --git a/graylog-plugin-archetype/pom.xml b/graylog-plugin-archetype/pom.xml index 00bc46eecfc8..5f4b9786f230 100644 --- a/graylog-plugin-archetype/pom.xml +++ b/graylog-plugin-archetype/pom.xml @@ -22,7 +22,7 @@ org.graylog graylog-project-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-project-parent diff --git a/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml b/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml index 19d5058d3fe9..0838e463a673 100644 --- a/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml +++ b/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml @@ -22,7 +22,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 graylog-plugin-web-parent diff --git a/graylog-plugin-parent/pom.xml b/graylog-plugin-parent/pom.xml index 3356bab9e2f5..e52101814b9e 100644 --- a/graylog-plugin-parent/pom.xml +++ b/graylog-plugin-parent/pom.xml @@ -26,7 +26,7 @@ org.graylog graylog-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 org.graylog.plugins diff --git a/graylog-project-parent/pom.xml b/graylog-project-parent/pom.xml index 2b55cd648d99..80b3509b4490 100644 --- a/graylog-project-parent/pom.xml +++ b/graylog-project-parent/pom.xml @@ -27,7 +27,7 @@ org.graylog graylog-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 graylog-project-parent diff --git a/graylog-storage-elasticsearch7/pom.xml b/graylog-storage-elasticsearch7/pom.xml index 7c9b1a4baf33..69d300be1505 100644 --- a/graylog-storage-elasticsearch7/pom.xml +++ b/graylog-storage-elasticsearch7/pom.xml @@ -23,7 +23,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-plugin-parent 4.0.0 diff --git a/graylog-storage-opensearch2/pom.xml b/graylog-storage-opensearch2/pom.xml index 277035d495ee..c191f184d177 100644 --- a/graylog-storage-opensearch2/pom.xml +++ b/graylog-storage-opensearch2/pom.xml @@ -23,7 +23,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-plugin-parent 4.0.0 diff --git a/graylog2-server/pom.xml b/graylog2-server/pom.xml index f0d1bf5d1060..ff25a7091918 100644 --- a/graylog2-server/pom.xml +++ b/graylog2-server/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 ../graylog-project-parent diff --git a/pom.xml b/pom.xml index a80a866a93f3..6f7fa87c9601 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ org.graylog graylog-parent - 6.1.0-SNAPSHOT + 6.1.0-alpha.7 pom Graylog Parent POM From 5c8414254e8ef0c543ef4b1c68f2ef8f926babae Mon Sep 17 00:00:00 2001 From: Jenkins Date: Fri, 16 Aug 2024 14:34:28 +0200 Subject: [PATCH 11/25] Bump package.json version to 6.1.0-SNAPSHOT --- graylog2-web-interface/manifests/package.json | 2 +- graylog2-web-interface/package.json | 2 +- graylog2-web-interface/packages/graylog-web-plugin/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graylog2-web-interface/manifests/package.json b/graylog2-web-interface/manifests/package.json index c7f624d4fe8c..5ee9df35fe99 100644 --- a/graylog2-web-interface/manifests/package.json +++ b/graylog2-web-interface/manifests/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-manifests", - "version": "6.1.0-alpha.7", + "version": "6.1.0-SNAPSHOT", "description": "Manifests needed to reuse vendor & shared bundles in graylog web interface plugins", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" diff --git a/graylog2-web-interface/package.json b/graylog2-web-interface/package.json index 121f94a114f2..3d0ca6778b29 100644 --- a/graylog2-web-interface/package.json +++ b/graylog2-web-interface/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-interface", - "version": "6.1.0-alpha.7", + "version": "6.1.0-SNAPSHOT", "description": "Graylog Web Interface", "author": "torch", "license": "SSPL-1.0", diff --git a/graylog2-web-interface/packages/graylog-web-plugin/package.json b/graylog2-web-interface/packages/graylog-web-plugin/package.json index cac0968b129a..a8b545c07f30 100644 --- a/graylog2-web-interface/packages/graylog-web-plugin/package.json +++ b/graylog2-web-interface/packages/graylog-web-plugin/package.json @@ -1,6 +1,6 @@ { "name": "graylog-web-plugin", - "version": "6.1.0-alpha.7", + "version": "6.1.0-SNAPSHOT", "description": "Helper code for streamlining Graylog web interface plugin development", "main": "index.js", "repository": { From a74ff7c7185b9addfcd6e5b38842bf94497703f1 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Fri, 16 Aug 2024 14:34:31 +0200 Subject: [PATCH 12/25] [graylog2-server] prepare for next development iteration --- data-node/pom.xml | 2 +- distribution/pom.xml | 2 +- full-backend-tests/pom.xml | 2 +- graylog-plugin-archetype/pom.xml | 2 +- graylog-plugin-parent/graylog-plugin-web-parent/pom.xml | 2 +- graylog-plugin-parent/pom.xml | 2 +- graylog-project-parent/pom.xml | 2 +- graylog-storage-elasticsearch7/pom.xml | 2 +- graylog-storage-opensearch2/pom.xml | 2 +- graylog2-server/pom.xml | 2 +- pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data-node/pom.xml b/data-node/pom.xml index f9c9165c1bde..3afbe102faab 100644 --- a/data-node/pom.xml +++ b/data-node/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-project-parent diff --git a/distribution/pom.xml b/distribution/pom.xml index 9ec854616a0b..3bd1d10c1b66 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -23,7 +23,7 @@ graylog-parent org.graylog - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT 4.0.0 diff --git a/full-backend-tests/pom.xml b/full-backend-tests/pom.xml index 23363f39c973..047b1ffd2df6 100644 --- a/full-backend-tests/pom.xml +++ b/full-backend-tests/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-project-parent diff --git a/graylog-plugin-archetype/pom.xml b/graylog-plugin-archetype/pom.xml index 5f4b9786f230..00bc46eecfc8 100644 --- a/graylog-plugin-archetype/pom.xml +++ b/graylog-plugin-archetype/pom.xml @@ -22,7 +22,7 @@ org.graylog graylog-project-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-project-parent diff --git a/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml b/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml index 0838e463a673..19d5058d3fe9 100644 --- a/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml +++ b/graylog-plugin-parent/graylog-plugin-web-parent/pom.xml @@ -22,7 +22,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT graylog-plugin-web-parent diff --git a/graylog-plugin-parent/pom.xml b/graylog-plugin-parent/pom.xml index e52101814b9e..3356bab9e2f5 100644 --- a/graylog-plugin-parent/pom.xml +++ b/graylog-plugin-parent/pom.xml @@ -26,7 +26,7 @@ org.graylog graylog-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT org.graylog.plugins diff --git a/graylog-project-parent/pom.xml b/graylog-project-parent/pom.xml index 80b3509b4490..2b55cd648d99 100644 --- a/graylog-project-parent/pom.xml +++ b/graylog-project-parent/pom.xml @@ -27,7 +27,7 @@ org.graylog graylog-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT graylog-project-parent diff --git a/graylog-storage-elasticsearch7/pom.xml b/graylog-storage-elasticsearch7/pom.xml index 69d300be1505..7c9b1a4baf33 100644 --- a/graylog-storage-elasticsearch7/pom.xml +++ b/graylog-storage-elasticsearch7/pom.xml @@ -23,7 +23,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-plugin-parent 4.0.0 diff --git a/graylog-storage-opensearch2/pom.xml b/graylog-storage-opensearch2/pom.xml index c191f184d177..277035d495ee 100644 --- a/graylog-storage-opensearch2/pom.xml +++ b/graylog-storage-opensearch2/pom.xml @@ -23,7 +23,7 @@ org.graylog.plugins graylog-plugin-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-plugin-parent 4.0.0 diff --git a/graylog2-server/pom.xml b/graylog2-server/pom.xml index ff25a7091918..f0d1bf5d1060 100644 --- a/graylog2-server/pom.xml +++ b/graylog2-server/pom.xml @@ -23,7 +23,7 @@ org.graylog graylog-project-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT ../graylog-project-parent diff --git a/pom.xml b/pom.xml index 6f7fa87c9601..a80a866a93f3 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ org.graylog graylog-parent - 6.1.0-alpha.7 + 6.1.0-SNAPSHOT pom Graylog Parent POM From e8b069e06778751fe5efa6f93bc0beee4a99d919 Mon Sep 17 00:00:00 2001 From: Zack King <91903901+kingzacko1@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:35:46 -0500 Subject: [PATCH 13/25] Change default mode for KafkaTransport to non-legacy (#20126) * Change default mode for KafkaTransport to non-legacy * Add default configuration value change to UPGRADING.md --- UPGRADING.md | 3 +++ changelog/unreleased/issue-19693.toml | 4 ++++ .../java/org/graylog2/inputs/transports/KafkaTransport.java | 5 ++--- 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/issue-19693.toml diff --git a/UPGRADING.md b/UPGRADING.md index 0b87afbae6f4..0aedbd28f50d 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -17,6 +17,9 @@ refactoring. Regular users should not change the setting. compatibility, the default value is still count-based (`500`). Previously configured count-based values are still supported. +- The legacy mode configuration setting of newly created Kafka based inputs has been changed to false. The default mode +will now use the high level consumer API that has been available since Kafka 1.x. + ## Java API Changes The following Java Code API changes have been made. diff --git a/changelog/unreleased/issue-19693.toml b/changelog/unreleased/issue-19693.toml new file mode 100644 index 000000000000..115c70ed7f3c --- /dev/null +++ b/changelog/unreleased/issue-19693.toml @@ -0,0 +1,4 @@ +type="c" +message="Changed default setting for Kafka transports to non-legacy mode." + +issues=["19693"] diff --git a/graylog2-server/src/main/java/org/graylog2/inputs/transports/KafkaTransport.java b/graylog2-server/src/main/java/org/graylog2/inputs/transports/KafkaTransport.java index 4d3f0aa6c9a5..7b0d4b91e5f9 100644 --- a/graylog2-server/src/main/java/org/graylog2/inputs/transports/KafkaTransport.java +++ b/graylog2-server/src/main/java/org/graylog2/inputs/transports/KafkaTransport.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; +import jakarta.inject.Named; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.InvalidOffsetException; @@ -66,8 +67,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.inject.Named; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -507,7 +506,7 @@ public ConfigurationRequest getRequestedConfiguration() { cr.addField(new BooleanField(CK_LEGACY, "Legacy mode", - true, + false, "Use old ZooKeeper-based consumer API. (Used before Graylog 3.3)", 10 )); From 494ae5805c986d8127beeed4084eedc0f172b3ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:56:24 +0200 Subject: [PATCH 14/25] Bump qs from 6.12.1 to 6.13.0 in /graylog2-web-interface (#20207) Bumps [qs](https://github.com/ljharb/qs) from 6.12.1 to 6.13.0. - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.12.1...v6.13.0) --- updated-dependencies: - dependency-name: qs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog2-web-interface/yarn.lock | 134 +++++++++++++++---------------- 1 file changed, 66 insertions(+), 68 deletions(-) diff --git a/graylog2-web-interface/yarn.lock b/graylog2-web-interface/yarn.lock index 53a70f850a35..c5716d4a4792 100644 --- a/graylog2-web-interface/yarn.lock +++ b/graylog2-web-interface/yarn.lock @@ -1756,25 +1756,25 @@ dependencies: "@jridgewell/trace-mapping" "0.3.9" -"@csstools/css-parser-algorithms@^2.7.1": - version "2.7.1" - resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-2.7.1.tgz#6d93a8f7d8aeb7cd9ed0868f946e46f021b6aa70" - integrity sha512-2SJS42gxmACHgikc1WGesXLIT8d/q2l0UFM7TaEeIzdFCE/FPMtTiizcPGGJtlPo2xuQzY09OhrLTzRxqJqwGw== +"@csstools/css-parser-algorithms@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.1.tgz#f14ade63bae5f6025ac85c7d03fe47a7ca0e58af" + integrity sha512-lSquqZCHxDfuTg/Sk2hiS0mcSFCEBuj49JfzPHJogDBT0mGCyY5A1AQzBWngitrp7i1/HAZpIgzF/VjhOEIJIg== -"@csstools/css-tokenizer@^2.4.1": - version "2.4.1" - resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-2.4.1.tgz#1d8b2e200197cf5f35ceb07ca2dade31f3a00ae8" - integrity sha512-eQ9DIktFJBhGjioABJRtUucoWR2mwllurfnM8LuNGAqX3ViZXaUchqk+1s7jjtkFiT9ySdACsFEA3etErkALUg== +"@csstools/css-tokenizer@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@csstools/css-tokenizer/-/css-tokenizer-3.0.1.tgz#9dd9b10084f3011290f96789598091e5bcb3c29a" + integrity sha512-UBqaiu7kU0lfvaP982/o3khfXccVlHPWp0/vwwiIgDF0GmqqqxoiXC/6FCjlS9u92f7CoEz6nXKQnrn1kIAkOw== -"@csstools/media-query-list-parser@^2.1.13": - version "2.1.13" - resolved "https://registry.yarnpkg.com/@csstools/media-query-list-parser/-/media-query-list-parser-2.1.13.tgz#f00be93f6bede07c14ddf51a168ad2748e4fe9e5" - integrity sha512-XaHr+16KRU9Gf8XLi3q8kDlI18d5vzKSKCY510Vrtc9iNR0NJzbY9hhTmwhzYZj/ZwGL4VmB3TA9hJW0Um2qFA== +"@csstools/media-query-list-parser@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@csstools/media-query-list-parser/-/media-query-list-parser-3.0.1.tgz#9474e08e6d7767cf68c56bf1581b59d203360cb0" + integrity sha512-HNo8gGD02kHmcbX6PvCoUuOQvn4szyB9ca63vZHKX5A81QytgDG4oxG4IaEfHTlEZSZ6MjPEMWIVU+zF2PZcgw== -"@csstools/selector-specificity@^3.1.1": - version "3.1.1" - resolved "https://registry.yarnpkg.com/@csstools/selector-specificity/-/selector-specificity-3.1.1.tgz#63085d2995ca0f0e55aa8b8a07d69bfd48b844fe" - integrity sha512-a7cxGcJ2wIlMFLlh8z2ONm+715QkPHiyJcxwQlKOz/03GPw1COpfhcmC9wm4xlZfp//jWHNNMwzjtqHXVWU9KA== +"@csstools/selector-specificity@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@csstools/selector-specificity/-/selector-specificity-4.0.0.tgz#7dfccb9df5499e627e7bfdbb4021a06813a45dba" + integrity sha512-189nelqtPd8++phaHNwYovKZI0FOzH1vQEE3QhHHkNIGrg5fSs9CbYP3RvfEH5geztnIA9Jwq91wyOIwAW5JIQ== "@cyclonedx/cyclonedx-library@^6.11.0": version "6.11.0" @@ -2934,10 +2934,10 @@ redux-thunk "^3.1.0" reselect "^5.1.0" -"@remix-run/router@1.19.0": - version "1.19.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.19.0.tgz#745dbffbce67f05386d57ca22c51dfd85c979593" - integrity sha512-zDICCLKEwbVYTS6TjYaWtHXxkdoUvD/QXvyVZjGCsWz5vyH7aFeONlPffPdW+Y/t6KT0MgXb2Mfjun9YpWN1dA== +"@remix-run/router@1.19.1": + version "1.19.1" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.19.1.tgz#984771bfd1de2715f42394c87fb716c1349e014f" + integrity sha512-S45oynt/WH19bHbIXjtli6QmwNYvaz+vtnubvNpNDvUOoA/OWh6j1OikIP3G+v5GHdxyC6EXoChG3HgYGEUfcg== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -6192,13 +6192,6 @@ debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.3.5: - version "4.3.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" - integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== - dependencies: - ms "2.1.2" - debug@^4.3.6: version "4.3.6" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" @@ -8877,7 +8870,7 @@ graphemer@^1.4.0: react "18.3.1" react-dom "18.2.0" react-router-bootstrap "0.26.3" - react-router-dom "6.26.0" + react-router-dom "6.26.1" reflux "0.2.13" styled-components "6.1.1" typescript "5.5.4" @@ -9383,6 +9376,11 @@ ignore@^5.3.1: resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== +ignore@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" + integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== + image-size@~0.5.0: version "0.5.5" resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c" @@ -12740,10 +12738,10 @@ postcss-modules-values@^4.0.0: dependencies: icss-utils "^5.0.0" -postcss-resolve-nested-selector@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.1.tgz#29ccbc7c37dedfac304e9fff0bf1596b3f6a0e4e" - integrity sha1-Kcy8fDfe36wwTp//C/FZaz9qDk4= +postcss-resolve-nested-selector@^0.1.6: + version "0.1.6" + resolved "https://registry.yarnpkg.com/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz#3d84dec809f34de020372c41b039956966896686" + integrity sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw== postcss-safe-parser@^7.0.0: version "7.0.0" @@ -12769,10 +12767,10 @@ postcss-selector-parser@^6.0.4: uniq "^1.0.1" util-deprecate "^1.0.2" -postcss-selector-parser@^6.1.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz#49694cb4e7c649299fea510a29fa6577104bcf53" - integrity sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ== +postcss-selector-parser@^6.1.2: + version "6.1.2" + resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.2.tgz#27ecb41fb0e3b6ba7a1ec84fff347f734c7929de" + integrity sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg== dependencies: cssesc "^3.0.0" util-deprecate "^1.0.2" @@ -12798,10 +12796,10 @@ postcss@^8.4.31, postcss@^8.4.33: picocolors "^1.0.0" source-map-js "^1.0.2" -postcss@^8.4.39: - version "8.4.39" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.39.tgz#aa3c94998b61d3a9c259efa51db4b392e1bde0e3" - integrity sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw== +postcss@^8.4.41: + version "8.4.41" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.41.tgz#d6104d3ba272d882fe18fc07d15dc2da62fa2681" + integrity sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ== dependencies: nanoid "^3.3.7" picocolors "^1.0.1" @@ -13008,9 +13006,9 @@ qs@6.11.0: side-channel "^1.0.4" qs@^6.3.0: - version "6.12.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.1.tgz#39422111ca7cbdb70425541cba20c7d7b216599a" - integrity sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ== + version "6.13.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" + integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: side-channel "^1.0.6" @@ -13367,20 +13365,20 @@ react-router-bootstrap@0.26.3: dependencies: prop-types "^15.7.2" -react-router-dom@6.26.0: - version "6.26.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.26.0.tgz#8debe13295c58605c04f93018d659a763245e58c" - integrity sha512-RRGUIiDtLrkX3uYcFiCIxKFWMcWQGMojpYZfcstc63A1+sSnVgILGIm9gNUA6na3Fm1QuPGSBQH2EMbAZOnMsQ== +react-router-dom@6.26.1: + version "6.26.1" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.26.1.tgz#a408892b41767a49dc94b3564b0e7d8e3959f623" + integrity sha512-veut7m41S1fLql4pLhxeSW3jlqs+4MtjRLj0xvuCEXsxusJCbs6I8yn9BxzzDX2XDgafrccY6hwjmd/bL54tFw== dependencies: - "@remix-run/router" "1.19.0" - react-router "6.26.0" + "@remix-run/router" "1.19.1" + react-router "6.26.1" -react-router@6.26.0: - version "6.26.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.26.0.tgz#d5af4c46835b202348ef2b7ddacd32a2db539fde" - integrity sha512-wVQq0/iFYd3iZ9H2l3N3k4PL8EEHcb0XlU2Na8nEwmiXgIUElEH6gaJDtUQxJ+JFzmIXaQjfdpcGWaM6IoQGxg== +react-router@6.26.1: + version "6.26.1" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.26.1.tgz#88c64837e05ffab6899a49df2a1484a22471e4ce" + integrity sha512-kIwJveZNwp7teQRI5QmwWo39A5bXRyqpH0COKKmPnyD2vBvDwgFXSqDUYtt1h+FEyfnE8eXr7oe0MxRzVwCcvQ== dependencies: - "@remix-run/router" "1.19.0" + "@remix-run/router" "1.19.1" react-select-event@5.5.1: version "5.5.1" @@ -15102,7 +15100,7 @@ styled-components@6.1.1: version "1.0.0" dependencies: postcss-styled-syntax "0.6.4" - stylelint "16.7.0" + stylelint "16.8.2" stylelint-config-recommended "14.0.1" stylelint-config-standard "36.0.1" stylelint-config-styled-components "0.1.1" @@ -15124,22 +15122,22 @@ stylelint-config-styled-components@0.1.1: resolved "https://registry.yarnpkg.com/stylelint-config-styled-components/-/stylelint-config-styled-components-0.1.1.tgz#b408388d7c687833ab4be4c4e6522d97d2827ede" integrity sha512-z5Xz/9GmvxO6e/DLzBMwkB85zHxEEjN6K7Cj80Bi+o/9vR9eS3GX3E9VuMnX9WLFYulqbqLtTapGGY28JBiy9Q== -stylelint@16.7.0: - version "16.7.0" - resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-16.7.0.tgz#5f6acf516aedecba7a6472ba0cc1ffc20e2be86b" - integrity sha512-Q1ATiXlz+wYr37a7TGsfvqYn2nSR3T/isw3IWlZQzFzCNoACHuGBb6xBplZXz56/uDRJHIygxjh7jbV/8isewA== +stylelint@16.8.2: + version "16.8.2" + resolved "https://registry.yarnpkg.com/stylelint/-/stylelint-16.8.2.tgz#7fda18b919a36e206e897417d4720baceb3af122" + integrity sha512-fInKATippQhcSm7AB+T32GpI+626yohrg33GkFT/5jzliUw5qhlwZq2UQQwgl3HsHrf09oeARi0ZwgY/UWEv9A== dependencies: - "@csstools/css-parser-algorithms" "^2.7.1" - "@csstools/css-tokenizer" "^2.4.1" - "@csstools/media-query-list-parser" "^2.1.13" - "@csstools/selector-specificity" "^3.1.1" + "@csstools/css-parser-algorithms" "^3.0.0" + "@csstools/css-tokenizer" "^3.0.0" + "@csstools/media-query-list-parser" "^3.0.0" + "@csstools/selector-specificity" "^4.0.0" "@dual-bundle/import-meta-resolve" "^4.1.0" balanced-match "^2.0.0" colord "^2.9.3" cosmiconfig "^9.0.0" css-functions-list "^3.2.2" css-tree "^2.3.1" - debug "^4.3.5" + debug "^4.3.6" fast-glob "^3.3.2" fastest-levenshtein "^1.0.16" file-entry-cache "^9.0.0" @@ -15147,7 +15145,7 @@ stylelint@16.7.0: globby "^11.1.0" globjoin "^0.1.4" html-tags "^3.3.1" - ignore "^5.3.1" + ignore "^5.3.2" imurmurhash "^0.1.4" is-plain-object "^5.0.0" known-css-properties "^0.34.0" @@ -15156,10 +15154,10 @@ stylelint@16.7.0: micromatch "^4.0.7" normalize-path "^3.0.0" picocolors "^1.0.1" - postcss "^8.4.39" - postcss-resolve-nested-selector "^0.1.1" + postcss "^8.4.41" + postcss-resolve-nested-selector "^0.1.6" postcss-safe-parser "^7.0.0" - postcss-selector-parser "^6.1.0" + postcss-selector-parser "^6.1.2" postcss-value-parser "^4.2.0" resolve-from "^5.0.0" string-width "^4.2.3" From 368cad39e30122f144b102c3aa88f55df0d8f9a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:56:56 +0200 Subject: [PATCH 15/25] Bump material-symbols from 0.22.0 to 0.22.1 in /graylog2-web-interface (#20206) Bumps [material-symbols](https://github.com/marella/material-symbols/tree/HEAD/material-symbols) from 0.22.0 to 0.22.1. - [Release notes](https://github.com/marella/material-symbols/releases) - [Commits](https://github.com/marella/material-symbols/commits/v0.22.1/material-symbols) --- updated-dependencies: - dependency-name: material-symbols dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog2-web-interface/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/graylog2-web-interface/yarn.lock b/graylog2-web-interface/yarn.lock index c5716d4a4792..523caeae6ae1 100644 --- a/graylog2-web-interface/yarn.lock +++ b/graylog2-web-interface/yarn.lock @@ -11415,9 +11415,9 @@ material-colors@^1.2.1: integrity sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg== material-symbols@^0.22.0: - version "0.22.0" - resolved "https://registry.yarnpkg.com/material-symbols/-/material-symbols-0.22.0.tgz#1e1b4c0e3f37e4e4768744a98cdb163536342ee0" - integrity sha512-SrfUYKNdOFhfpQ84slrcLSFIbFZQ24IuF17jaM6uS2pIyVOd1rnzyWqjK8vGzpuzFO/1gYDH2aJb8b1TPC0Ggw== + version "0.22.1" + resolved "https://registry.yarnpkg.com/material-symbols/-/material-symbols-0.22.1.tgz#a7254674a0ee4d4ad6667f5702d5fdbf9761dc8a" + integrity sha512-MWZB/53EMWzL3RYpr7plQRAWuqW73C/21PhvSwh/dptEGUk7SpG8h45eCqe37wfQoq6YZvuYfwLEYGfK2dgUqQ== math-log2@^1.0.1: version "1.0.1" From 78d4fd779ae2e10eb170addfc5ad77d5e867d59b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:03:27 +0200 Subject: [PATCH 16/25] Bump org.vafer:jdeb from 1.10 to 1.11 (#20204) Bumps [org.vafer:jdeb](https://github.com/tcurdt/jdeb) from 1.10 to 1.11. - [Changelog](https://github.com/tcurdt/jdeb/blob/master/HISTORY.md) - [Commits](https://github.com/tcurdt/jdeb/compare/jdeb-1.10...jdeb-1.11) --- updated-dependencies: - dependency-name: org.vafer:jdeb dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- graylog-plugin-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graylog-plugin-parent/pom.xml b/graylog-plugin-parent/pom.xml index 3356bab9e2f5..8fc28294e8de 100644 --- a/graylog-plugin-parent/pom.xml +++ b/graylog-plugin-parent/pom.xml @@ -40,7 +40,7 @@ ${project.build.directory}/web/build - 1.10 + 1.11 2.2.0 From cdad0e0ed348832dcd84239c5e4ee3998685af27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:05:31 +0200 Subject: [PATCH 17/25] Bump io.dropwizard.metrics:metrics-bom from 4.2.26 to 4.2.27 (#20203) Bumps [io.dropwizard.metrics:metrics-bom](https://github.com/dropwizard/metrics) from 4.2.26 to 4.2.27. - [Release notes](https://github.com/dropwizard/metrics/releases) - [Commits](https://github.com/dropwizard/metrics/compare/v4.2.26...v4.2.27) --- updated-dependencies: - dependency-name: io.dropwizard.metrics:metrics-bom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a80a866a93f3..53430ae38ecf 100644 --- a/pom.xml +++ b/pom.xml @@ -150,7 +150,7 @@ 0.9.0.1-7 2.23.1 9.11.1 - 4.2.26 + 4.2.27 5.1.3 4.11.0 0.13 From 14149f0a045f3171a2161719db893fd5f164ca26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:07:17 +0200 Subject: [PATCH 18/25] Bump org.apache.maven.plugins:maven-failsafe-plugin from 3.3.1 to 3.4.0 (#20201) Bumps [org.apache.maven.plugins:maven-failsafe-plugin](https://github.com/apache/maven-surefire) from 3.3.1 to 3.4.0. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-3.3.1...surefire-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-failsafe-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53430ae38ecf..8980a8813532 100644 --- a/pom.xml +++ b/pom.xml @@ -604,7 +604,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.3.1 + 3.4.0 org.codehaus.mojo From c9d174b8c6432e875da880774b40c71eaf13720a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:07:32 +0200 Subject: [PATCH 19/25] Bump org.apache.maven.plugins:maven-surefire-plugin from 3.3.1 to 3.4.0 (#20202) Bumps [org.apache.maven.plugins:maven-surefire-plugin](https://github.com/apache/maven-surefire) from 3.3.1 to 3.4.0. - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-3.3.1...surefire-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-surefire-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8980a8813532..7153e6930661 100644 --- a/pom.xml +++ b/pom.xml @@ -599,7 +599,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.3.1 + 3.4.0 org.apache.maven.plugins From d9468ef12e26f68a92429e4cfddc870bcc7ee40d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:08:33 +0200 Subject: [PATCH 20/25] Bump com.google.guava:guava from 33.2.1-jre to 33.3.0-jre (#20205) Bumps [com.google.guava:guava](https://github.com/google/guava) from 33.2.1-jre to 33.3.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7153e6930661..9f205a7b3e62 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ 0.1.9-graylog-3 1.65.1 2.0.0 - 33.2.1-jre + 33.3.0-jre 7.0.0 2.2.2 3.0 From f0d129f5d5e186a0875161f858f35297701f40ee Mon Sep 17 00:00:00 2001 From: Dennis Oelkers Date: Mon, 19 Aug 2024 14:29:48 +0200 Subject: [PATCH 21/25] Fixing `chroma-js` imports. (#20209) --- .../src/views/components/sidebar/highlighting/Scale.ts | 2 +- .../logic/views/formatting/highlighting/HighlightingColor.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/graylog2-web-interface/src/views/components/sidebar/highlighting/Scale.ts b/graylog2-web-interface/src/views/components/sidebar/highlighting/Scale.ts index c6ec32fd225f..b9c5b85fbfc8 100644 --- a/graylog2-web-interface/src/views/components/sidebar/highlighting/Scale.ts +++ b/graylog2-web-interface/src/views/components/sidebar/highlighting/Scale.ts @@ -15,7 +15,7 @@ * . */ import { scales } from 'plotly.js/src/components/colorscale'; -import * as chroma from 'chroma-js'; +import chroma from 'chroma-js'; const plotlyScaleToChroma = (plotlyScale: Array<[domain: number, color: string]>) => { const domains = plotlyScale.map(([domain]) => domain); diff --git a/graylog2-web-interface/src/views/logic/views/formatting/highlighting/HighlightingColor.ts b/graylog2-web-interface/src/views/logic/views/formatting/highlighting/HighlightingColor.ts index 9a277f7ae2c0..eea5e363f5e1 100644 --- a/graylog2-web-interface/src/views/logic/views/formatting/highlighting/HighlightingColor.ts +++ b/graylog2-web-interface/src/views/logic/views/formatting/highlighting/HighlightingColor.ts @@ -14,7 +14,7 @@ * along with this program. If not, see * . */ -import type * as chroma from 'chroma-js'; +import type chroma from 'chroma-js'; import scaleForGradient from 'views/components/sidebar/highlighting/Scale'; From a38aeef7d3f2e5a24cd6386a3fe41abf2b80a70a Mon Sep 17 00:00:00 2001 From: luk-kaminski Date: Mon, 19 Aug 2024 14:49:27 +0200 Subject: [PATCH 22/25] Fixing NPE in XLSXWriter. Adding missing else clauses. (#20210) --- .../search/searchtypes/export/XLSXWriter.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/export/XLSXWriter.java b/graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/export/XLSXWriter.java index be90bb54fdbb..1ec9516e6f6a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/export/XLSXWriter.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/views/search/searchtypes/export/XLSXWriter.java @@ -59,20 +59,22 @@ public static void createWorksheetFor(final XSSFWorkbook wb, final String sheetN cell.setCellValue(d); } else if(rawData instanceof Boolean b) { cell.setCellValue(b); - } if(rawData instanceof Date d) { + } else if (rawData instanceof Date d) { cell.setCellValue(d); - } if(rawData instanceof LocalDateTime ldt) { + } else if (rawData instanceof LocalDateTime ldt) { cell.setCellValue(ldt); - } if(rawData instanceof Calendar cal) { + } else if (rawData instanceof Calendar cal) { cell.setCellValue(cal); - } if(rawData instanceof String s) { + } else if (rawData instanceof String s) { cell.setCellValue(s); - } if(rawData instanceof RichTextString rts) { + } else if (rawData instanceof RichTextString rts) { cell.setCellValue(rts); - } if(rawData instanceof LocalDate ld) { + } else if (rawData instanceof LocalDate ld) { cell.setCellValue(ld); - } else { + } else if (rawData != null) { cell.setCellValue(rawData.toString()); + } else { + cell.setBlank(); } } } From d199e963c2297a5b2c506f5f6f3cd92f0cd0236e Mon Sep 17 00:00:00 2001 From: Patrick Mann Date: Mon, 19 Aug 2024 15:02:45 +0200 Subject: [PATCH 23/25] Supporting changes for draw-down licenses #8123 (#20137) * document new APIs * Add license management page to docs helper --- UPGRADING.md | 26 ++++++++++--------- graylog2-web-interface/src/util/DocsHelper.ts | 1 + 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 0aedbd28f50d..e5c8303a7485 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -33,18 +33,20 @@ The following Java Code API changes have been made. The following REST API changes have been made. -| Endpoint | Description | -|------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| -| `PUT /system/indices/index_set_defaults` | This endpoint now expects an index set template id as payload. The values of the index set template are used as default values. | -| `GET licenses/{licenseId}` | deprecated | -| `GET licenses` | deprecated | -| `GET licenses/status` | deprecated | -| `GET licenses/status/active` | New: Show status for currently active license | -| `GET licenses/validity/for-subject` | Check for valid license for given subject | -| `GET licenses/status/for-subject` | deprecated | -| `DELETE licenses/{licenseId}` | When called with a contract ID it will delete the contract and all associated licenses | -| `GET licenses/traffic-remaining` | Get the time series data for remaining provisioned traffic | -| `GET licenses/metrics` | Get the stats for consumed and remaining provisioned traffic | +| Endpoint | Description | +|--------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| +| `PUT /system/indices/index_set_defaults` | This endpoint now expects an index set template id as payload. The values of the index set template are used as default values. | +| `GET licenses/{licenseId}` | deprecated | +| `GET licenses` | deprecated | +| `GET licenses/status` | deprecated | +| `GET licenses/status/active` | New: Show status for currently active license | +| `GET licenses/validity/for-subject` | Check for valid license for given subject | +| `GET licenses/status/for-subject` | deprecated | +| `DELETE licenses/{licenseId}` | When called with a contract ID it will delete the contract and all associated licenses | +| `GET licenses/traffic-remaining` | Get the time series data for remaining provisioned traffic | +| `GET licenses/metrics` | Get the stats for consumed and remaining provisioned traffic | +| `GET licenses/traffic-threshold` | Get info about license traffic threshold warning | +| `PUT licenses/traffic-threshold/acknowledgement` | Acknowledge current traffic threshold warning | ## Deprecated Inputs diff --git a/graylog2-web-interface/src/util/DocsHelper.ts b/graylog2-web-interface/src/util/DocsHelper.ts index caf92d843b5f..1ca7073de119 100644 --- a/graylog2-web-interface/src/util/DocsHelper.ts +++ b/graylog2-web-interface/src/util/DocsHelper.ts @@ -37,6 +37,7 @@ const docsHelper = { INDEXER_FAILURES: 'indexer-failures', INDEX_MODEL: 'index-model', LICENSE: 'license', + LICENSE_MANAGEMENT: 'license-management', LOAD_BALANCERS: 'load-balancers', LOOKUPTABLES: 'lookuptables', OPERATIONS_CHANGELOG: 'changelog-graylog', From 8bdd5f0108f6e1543b077369ac3e29ae9f1a6555 Mon Sep 17 00:00:00 2001 From: simonychuang Date: Mon, 19 Aug 2024 09:12:51 -0400 Subject: [PATCH 24/25] Force select menu to open on bottom to prevent menu being cut off in security perspective modal (#20182) --- .../event-definition-form/AddNotificationForm.jsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graylog2-web-interface/src/components/event-definitions/event-definition-form/AddNotificationForm.jsx b/graylog2-web-interface/src/components/event-definitions/event-definition-form/AddNotificationForm.jsx index 986e0b900d20..eb4a5840a70b 100644 --- a/graylog2-web-interface/src/components/event-definitions/event-definition-form/AddNotificationForm.jsx +++ b/graylog2-web-interface/src/components/event-definitions/event-definition-form/AddNotificationForm.jsx @@ -142,7 +142,8 @@ class AddNotificationForm extends React.Component { placeholder="Select Notification" onChange={this.handleSelectNotificationChange} options={this.formatNotifications(notifications)} - value={selectedNotification} /> + value={selectedNotification} + menuPlacement="bottom" /> Select a Notification to use on Alerts of this kind or create a new Notification that you can later use in other Alerts. From 865a36d25e621b600f523813705044cf77704741 Mon Sep 17 00:00:00 2001 From: Zack King Date: Mon, 19 Aug 2024 08:45:28 -0500 Subject: [PATCH 25/25] Add enterprise issue to changelog --- changelog/unreleased/pr-20110.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog/unreleased/pr-20110.toml b/changelog/unreleased/pr-20110.toml index 00a6717495e3..f8c1a286322a 100644 --- a/changelog/unreleased/pr-20110.toml +++ b/changelog/unreleased/pr-20110.toml @@ -1,4 +1,5 @@ type = "a" message = "Added categories to Streams to allow Illuminate content to be scoped to multiple products." +issues = ["graylog-plugin-enterprise#7945"] pulls = ["20110"]