From 214c00842f8e50c75e323e2537f7125a9aef21d5 Mon Sep 17 00:00:00 2001 From: gskrobisz Date: Thu, 12 Sep 2024 09:52:36 +0200 Subject: [PATCH] [NU-1806] Add deployment activity parameters --- .../api/definition/FixedExpressionValue.scala | 5 +- .../api/editor/FixedValuesEditorMode.java | 15 ++ .../api/component/NodesDeploymentData.scala | 23 +-- .../api/definition/ParameterEditor.scala | 40 +++- .../engine/api/process/Source.scala | 11 +- designer/client/src/actions/actionTypes.ts | 1 + designer/client/src/actions/nk/process.ts | 10 + .../editors/expression/FixedValuesEditor.tsx | 24 ++- .../fragment-input-definition/item/types.ts | 1 + .../modals/ActivityCommentTextField.tsx | 21 +++ .../src/components/modals/ActivityHeader.tsx | 28 +++ .../components/modals/ActivityProperty.tsx | 49 +++++ .../modals/AdvancedParametersSection.tsx | 26 +++ .../CalculateCounts/CalculateCountsDialog.tsx | 2 +- .../components/modals/CustomActionDialog.tsx | 28 ++- .../components/modals/DeployProcessDialog.tsx | 92 ++++++--- .../modals/GenerateTestDataDialog.tsx | 2 +- .../components/modals/SaveProcessDialog.tsx | 18 +- .../toolbars/process/buttons/SaveButton.tsx | 4 +- .../buttons/CancelDeployButton.tsx | 2 +- .../scenarioActions/buttons/DeployButton.tsx | 9 +- designer/client/src/http/HttpService.ts | 24 ++- designer/client/src/reducers/graph/reducer.ts | 6 + designer/client/src/reducers/graph/types.ts | 2 + .../client/src/reducers/selectors/graph.ts | 2 + designer/client/src/types/activity.ts | 19 ++ .../src/windowManager/PromptContent.tsx | 9 +- .../restmodel/definition/package.scala | 7 + .../ui/api/ActivityInfoResources.scala | 45 +++++ .../ui/api/DeploymentApiHttpService.scala | 5 +- .../ui/api/ManagementApiHttpService.scala | 12 +- .../ui/api/ManagementResources.scala | 20 +- .../description/DeploymentApiEndpoints.scala | 15 +- .../ManagementApiEndpoints.scala | 14 +- .../api/description/NodesApiEndpoints.scala | 3 +- .../newactivity/ActivityInfoService.scala | 59 ++++++ .../server/AkkaHttpBasedRouteProvider.scala | 12 +- .../test/base/it/NuResourcesTest.scala | 6 +- .../ui/api/ActivityInfoResourcesSpec.scala | 88 +++++++++ .../ComponentApiHttpServiceBusinessSpec.scala | 1 + .../ComponentApiHttpServiceSecuritySpec.scala | 4 + .../ui/api/DefinitionResourcesSpec.scala | 7 +- .../ui/api/ManagementResourcesSpec.scala | 12 +- .../ui/api/TestingApiHttpServiceSpec.scala | 10 +- docs-internal/api/nu-designer-openapi.yaml | 178 +++++++++--------- .../api/process/FlinkCustomNodeContext.scala | 2 +- .../flink/util/source/CollectionSource.scala | 10 +- .../flink/table/source/TableSourceTest.scala | 9 +- .../aggregate/AggregateHelper.java | 29 +-- .../flink/table/source/TableSource.scala | 5 +- .../kafka/source/flink/FlinkKafkaSource.scala | 43 ++++- .../sample/DevProcessConfigCreator.scala | 14 +- .../sample/source/BoundedSource.scala | 95 +++++++++- .../src/universal/conf/dev-application.conf | 4 +- .../activity/ActivityInfoProvider.scala | 11 ++ .../CommonModelDataInfoProvider.scala | 40 ++++ .../ModelDataActivityInfoProvider.scala | 54 ++++++ .../EditorBasedLanguageDeterminer.scala | 2 +- ...bleValuesBasedDefaultValueDeterminer.scala | 6 +- .../EditorBasedValidatorExtractor.scala | 2 +- .../test/ModelDataTestInfoProvider.scala | 18 +- 61 files changed, 1049 insertions(+), 266 deletions(-) create mode 100644 components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java create mode 100644 designer/client/src/components/modals/ActivityCommentTextField.tsx create mode 100644 designer/client/src/components/modals/ActivityHeader.tsx create mode 100644 designer/client/src/components/modals/ActivityProperty.tsx create mode 100644 designer/client/src/components/modals/AdvancedParametersSection.tsx create mode 100644 designer/client/src/types/activity.ts create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala rename designer/server/src/main/scala/pl/touk/nussknacker/ui/api/{ => description}/ManagementApiEndpoints.scala (87%) create mode 100644 designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala create mode 100644 designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala create mode 100644 scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala diff --git a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala index dbfce58b797..a595e8b88ea 100644 --- a/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala +++ b/common-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/FixedExpressionValue.scala @@ -2,8 +2,9 @@ package pl.touk.nussknacker.engine.api.definition import io.circe.generic.JsonCodec -@JsonCodec case class FixedExpressionValue(expression: String, label: String) +@JsonCodec case class FixedExpressionValue(expression: String, label: String, hintText: Option[String] = None) object FixedExpressionValue { - val nullFixedValue: FixedExpressionValue = FixedExpressionValue("", "") + def apply(expression: String, label: String): FixedExpressionValue = FixedExpressionValue(expression, label, None) + val nullFixedValue: FixedExpressionValue = FixedExpressionValue("", "") } diff --git a/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java b/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java new file mode 100644 index 00000000000..99ab6778589 --- /dev/null +++ b/components-api/src/main/java/pl/touk/nussknacker/engine/api/editor/FixedValuesEditorMode.java @@ -0,0 +1,15 @@ +package pl.touk.nussknacker.engine.api.editor; + +public enum FixedValuesEditorMode { + LIST, RADIO; + + public static FixedValuesEditorMode fromName(String name) { + switch (name) { + case "LIST": + return LIST; + case "RADIO": + default: + return RADIO; + } + } +} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala index f032a97c297..f43126590f5 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/component/NodesDeploymentData.scala @@ -1,13 +1,18 @@ package pl.touk.nussknacker.engine.api.component -import io.circe.generic.extras.semiauto.{deriveUnwrappedDecoder, deriveUnwrappedEncoder} import io.circe.{Decoder, Encoder} import pl.touk.nussknacker.engine.api.NodeId +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData final case class NodesDeploymentData(dataByNodeId: Map[NodeId, NodeDeploymentData]) object NodesDeploymentData { + // Raw deployment parameters (name -> value) that are used as additional node configuration during deployment. + // Each node can be provided with dedicated set of parameters. + // TODO: consider replacing NodeDeploymentData with Json + type NodeDeploymentData = Map[String, String] + val empty: NodesDeploymentData = NodesDeploymentData(Map.empty) implicit val nodesDeploymentDataEncoder: Encoder[NodesDeploymentData] = Encoder @@ -18,19 +23,3 @@ object NodesDeploymentData { Decoder.decodeMap[NodeId, NodeDeploymentData].map(NodesDeploymentData(_)) } - -sealed trait NodeDeploymentData - -final case class SqlFilteringExpression(sqlExpression: String) extends NodeDeploymentData - -object NodeDeploymentData { - - implicit val nodeDeploymentDataEncoder: Encoder[NodeDeploymentData] = - deriveUnwrappedEncoder[SqlFilteringExpression].contramap { case sqlExpression: SqlFilteringExpression => - sqlExpression - } - - implicit val nodeDeploymentDataDecoder: Decoder[NodeDeploymentData] = - deriveUnwrappedDecoder[SqlFilteringExpression].map(identity) - -} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala index 83cc7259019..4170766d0d3 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/definition/ParameterEditor.scala @@ -2,8 +2,10 @@ package pl.touk.nussknacker.engine.api.definition import io.circe.generic.JsonCodec import io.circe.generic.extras.ConfiguredJsonCodec -import io.circe.{Decoder, Encoder, Json} +import io.circe.generic.semiauto.deriveEncoder +import io.circe.{Decoder, Encoder, HCursor, Json} import pl.touk.nussknacker.engine.api.CirceUtil._ +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode import pl.touk.nussknacker.engine.api.editor.DualEditorMode import java.time.temporal.ChronoUnit @@ -73,8 +75,10 @@ object PeriodParameterEditor { */ case object CronParameterEditor extends SimpleParameterEditor -@JsonCodec case class FixedValuesParameterEditor(possibleValues: List[FixedExpressionValue]) - extends SimpleParameterEditor +case class FixedValuesParameterEditor( + possibleValues: List[FixedExpressionValue], + mode: FixedValuesEditorMode = FixedValuesEditorMode.LIST +) extends SimpleParameterEditor @JsonCodec case class FixedValuesWithIconParameterEditor(possibleValues: List[FixedExpressionValueWithIcon]) extends SimpleParameterEditor @@ -100,3 +104,33 @@ object DualParameterEditor { } } + +object FixedValuesParameterEditor { + def apply(possibleValues: List[FixedExpressionValue]): FixedValuesParameterEditor = + FixedValuesParameterEditor(possibleValues, mode = FixedValuesEditorMode.LIST) + + implicit val fixedValuesEditorModeEncoder: Encoder[FixedValuesEditorMode] = new Encoder[FixedValuesEditorMode] { + override def apply(a: FixedValuesEditorMode): Json = Encoder.encodeString(a.name()) + } + + implicit val fixedValuesEditorModeDecoder: Decoder[FixedValuesEditorMode] = + Decoder.decodeString.emapTry(name => Try(FixedValuesEditorMode.fromName(name))) + + implicit val fixedValuesParameterEditorEncoder: Encoder[FixedValuesParameterEditor] = + deriveEncoder[FixedValuesParameterEditor] + + implicit val fixedValuesParameterEditorDecoder: Decoder[FixedValuesParameterEditor] = { (c: HCursor) => + { + for { + possibleValues <- c.downField("possibleValues").as[List[FixedExpressionValue]] + modeOpt <- c.downField("mode").as[Option[String]] + } yield { + FixedValuesParameterEditor( + possibleValues, + modeOpt.map(FixedValuesEditorMode.fromName).getOrElse(FixedValuesEditorMode.LIST) + ) + } + } + } + +} diff --git a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala index dbfb9904573..0a213e5bc6b 100644 --- a/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala +++ b/components-api/src/main/scala/pl/touk/nussknacker/engine/api/process/Source.scala @@ -1,7 +1,7 @@ package pl.touk.nussknacker.engine.api.process import pl.touk.nussknacker.engine.api.component.Component._ -import pl.touk.nussknacker.engine.api.component.{Component, ProcessingMode} +import pl.touk.nussknacker.engine.api.component.{Component, ParameterConfig, ProcessingMode} import pl.touk.nussknacker.engine.api.context.ContextTransformation import pl.touk.nussknacker.engine.api.definition.{Parameter, WithExplicitTypesToExtract} import pl.touk.nussknacker.engine.api.parameter.ParameterName @@ -49,6 +49,15 @@ trait TestWithParametersSupport[+T] { self: Source => def parametersToTestData(params: Map[ParameterName, AnyRef]): T } +/** + * Used to define Source parameters for each activity + * e.g. + * {"DEPLOY": { "parametername": ...parameter configuration... } + */ +trait WithActivityParameters { self: Source => + def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] +} + /** * [[pl.touk.nussknacker.engine.api.process.SourceFactory]] has to have method annotated with [[pl.touk.nussknacker.engine.api.MethodToInvoke]] * that returns [[pl.touk.nussknacker.engine.api.process.Source]] diff --git a/designer/client/src/actions/actionTypes.ts b/designer/client/src/actions/actionTypes.ts index 306aaea65fc..c0571599610 100644 --- a/designer/client/src/actions/actionTypes.ts +++ b/designer/client/src/actions/actionTypes.ts @@ -21,6 +21,7 @@ export type ActionTypes = | "PROCESS_RENAME" | "EDIT_LABELS" | "SHOW_METRICS" + | "UPDATE_ACTIVITY_PARAMETERS" | "UPDATE_TEST_CAPABILITIES" | "UPDATE_TEST_FORM_PARAMETERS" | "DISPLAY_PROCESS" diff --git a/designer/client/src/actions/nk/process.ts b/designer/client/src/actions/nk/process.ts index d7209ee2de0..b3e575917d9 100644 --- a/designer/client/src/actions/nk/process.ts +++ b/designer/client/src/actions/nk/process.ts @@ -36,6 +36,16 @@ export function loadProcessState(processName: ProcessName, processVersionId: num ); } +export function fetchActivityParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { + return (dispatch) => + HttpService.getActivityParameters(processName, scenarioGraph).then(({ data }) => { + dispatch({ + type: "UPDATE_ACTIVITY_PARAMETERS", + activityParameters: data, + }); + }); +} + export function fetchTestFormParameters(processName: ProcessName, scenarioGraph: ScenarioGraph) { return (dispatch) => HttpService.getTestFormParameters(processName, scenarioGraph).then(({ data }) => { diff --git a/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx b/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx index 1499b50cee6..e8e98113f33 100644 --- a/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx +++ b/designer/client/src/components/graph/node-modal/editors/expression/FixedValuesEditor.tsx @@ -5,7 +5,7 @@ import { ExpressionObj } from "./types"; import { isEmpty } from "lodash"; import { cx } from "@emotion/css"; import { selectStyled } from "../../../../../stylesheets/SelectStyled"; -import { Stack, styled, Typography, useTheme } from "@mui/material"; +import { FormControlLabel, Radio, RadioGroup, Stack, styled, Typography, useTheme } from "@mui/material"; import { ExtendedEditor } from "./Editor"; import { FieldError } from "../Validators"; import { FixedValuesOption } from "../../fragment-input-definition/item"; @@ -26,6 +26,7 @@ interface Option { label: string; value: string; icon: string | null; + hintText: string | null; } function getOptions(values: FixedValuesOption[]): Option[] { @@ -33,19 +34,26 @@ function getOptions(values: FixedValuesOption[]): Option[] { value: value.expression, label: value.label, icon: value.icon, + hintText: value.hintText, })); } +enum FixedValuesEditorMode { + LIST = "LIST", + RADIO = "RADIO", +} + export const FixedValuesEditor: ExtendedEditor = (props: Props) => { const handleCurrentOption = (expressionObj: ExpressionObj, options: Option[]): Option => { return ( (expressionObj && options.find((option) => option.value === expressionObj.expression)) || // current value with label taken from options - (expressionObj && { value: expressionObj.expression, label: expressionObj.expression, icon: null }) || // current value is no longer valid option? Show it anyway, let user know. Validation should take care + (expressionObj && { value: expressionObj.expression, label: expressionObj.expression, icon: null, hintText: null }) || // current value is no longer valid option? Show it anyway, let user know. Validation should take care null ); // just leave undefined and let the user explicitly select one }; const { expressionObj, readOnly, onValueChange, className, showValidation, editorConfig, fieldErrors } = props; + const mode = FixedValuesEditorMode[editorConfig.mode || "LIST"]; const options = getOptions(editorConfig.possibleValues); const currentOption = handleCurrentOption(expressionObj, options); const theme = useTheme(); @@ -58,7 +66,17 @@ export const FixedValuesEditor: ExtendedEditor = (props: Props) => { const { control, input, valueContainer, singleValue, menuPortal, menu, menuList, menuOption, indicatorSeparator, dropdownIndicator } = selectStyled(theme); - return ( + return mode == FixedValuesEditorMode.RADIO ? ( +
+ onValueChange(event.target.value)}> + {options.map((option: Option) => { + const label = option.value === "" ? `${option.value} (default)` : option.value; + return } label={label} />; + })} + + {currentOption.hintText ? {currentOption.hintText} : null} +
+ ) : (
( + +))({ + flexDirection: "column", + ".MuiFormLabel-root": { + margin: 0, + flexDirection: "column", + }, +}); diff --git a/designer/client/src/components/modals/ActivityHeader.tsx b/designer/client/src/components/modals/ActivityHeader.tsx new file mode 100644 index 00000000000..0829d9a22e9 --- /dev/null +++ b/designer/client/src/components/modals/ActivityHeader.tsx @@ -0,0 +1,28 @@ +import { useSelector } from "react-redux"; +import { getProcessName } from "../../reducers/selectors/graph"; +import { Typography } from "@mui/material"; +import React from "react"; +import ProcessDialogWarnings from "./ProcessDialogWarnings"; + +interface Props { + title: string; + displayWarnings?: boolean; +} + +export function ActivityHeader(props: Props): JSX.Element { + const processName = useSelector(getProcessName); + return ( + <> + + {props.title} + + + {processName} + + {props.displayWarnings && } + + ); +} diff --git a/designer/client/src/components/modals/ActivityProperty.tsx b/designer/client/src/components/modals/ActivityProperty.tsx new file mode 100644 index 00000000000..c401d025e16 --- /dev/null +++ b/designer/client/src/components/modals/ActivityProperty.tsx @@ -0,0 +1,49 @@ +import { ExpressionLang } from "../graph/node-modal/editors/expression/types"; +import React, { useCallback } from "react"; +import { FieldLabel } from "../graph/node-modal/FieldLabel"; +import { getValidationErrorsForField } from "../graph/node-modal/editors/Validators"; +import { ActivityNodeParameters, ActivityParameterConfig } from "../../types/activity"; +import { NodesDeploymentData } from "../../http/HttpService"; +import { NodeValidationError } from "../../types"; +import { default as EditableEditor } from "../graph/node-modal/editors/EditableEditor"; + +interface Props { + nodeName: string; + propertyName: string; + propertyConfig: ActivityParameterConfig; + nodesData: NodesDeploymentData; + onChange: ( + nodeId: string, + property: K, + newValue: ActivityNodeParameters["parameters"][K], + defaultValue?: ActivityNodeParameters["parameters"][K], + ) => void; + errors: NodeValidationError[]; +} + +export function ActivityProperty(props: Props): JSX.Element { + const { nodeName, propertyName, propertyConfig, errors, nodesData, onChange } = props; + + const current = nodesData[nodeName][propertyName] || ""; + const expressionObj = { expression: current, value: current, language: ExpressionLang.String }; + const onValueChange = useCallback((newValue) => onChange(nodeName, propertyName, newValue), [onChange, nodeName, propertyName]); + + return ( + ( + + )} + readOnly={false} + showSwitch={false} + showValidation={true} + //ScenarioProperties do not use any variables + variableTypes={{}} + fieldErrors={getValidationErrorsForField(errors, propertyName)} + /> + ); +} diff --git a/designer/client/src/components/modals/AdvancedParametersSection.tsx b/designer/client/src/components/modals/AdvancedParametersSection.tsx new file mode 100644 index 00000000000..4a5a3ebc0a5 --- /dev/null +++ b/designer/client/src/components/modals/AdvancedParametersSection.tsx @@ -0,0 +1,26 @@ +import React, { PropsWithChildren } from "react"; +import Accordion from "@mui/material/Accordion"; +import AccordionSummary from "@mui/material/AccordionSummary"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import { Typography } from "@mui/material"; +import AccordionDetails from "@mui/material/AccordionDetails"; + +interface Props { + nodeId: string; +} + +export function AdvancedParametersSection({ children, nodeId }: PropsWithChildren): JSX.Element { + return ( + + } + aria-controls={`${nodeId}-content`} + id={`${nodeId}-header`} + sx={{ flexDirection: "row-reverse", border: 0 }} + > + {nodeId} + + {children} + + ); +} diff --git a/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx b/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx index 7391493f8c5..3a2f045e048 100644 --- a/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx +++ b/designer/client/src/components/modals/CalculateCounts/CalculateCountsDialog.tsx @@ -59,7 +59,7 @@ export function CountsDialog({ children, ...props }: PropsWithChildren { await confirm(); diff --git a/designer/client/src/components/modals/CustomActionDialog.tsx b/designer/client/src/components/modals/CustomActionDialog.tsx index 8e16bf38c66..031f28f31e7 100644 --- a/designer/client/src/components/modals/CustomActionDialog.tsx +++ b/designer/client/src/components/modals/CustomActionDialog.tsx @@ -7,7 +7,7 @@ import { loadProcessState } from "../../actions/nk"; import HttpService, { CustomActionValidationRequest } from "../../http/HttpService"; import { CustomAction, NodeValidationError } from "../../types"; import { UnknownRecord } from "../../types/common"; -import { WindowContent, WindowKind } from "../../windowManager"; +import { PromptContent, WindowKind } from "../../windowManager"; import { ChangeableValue } from "../ChangeableValue"; import { editors, ExtendedEditor, SimpleEditor } from "../graph/node-modal/editors/expression/Editor"; import { ExpressionLang } from "../graph/node-modal/editors/expression/types"; @@ -18,8 +18,9 @@ import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; import { nodeValue } from "../graph/node-modal/NodeDetailsContent/NodeTableStyled"; import { getValidationErrorsForField } from "../graph/node-modal/editors/Validators"; import { getFeatureSettings } from "../../reducers/selectors/settings"; -import CommentInput from "../comment/CommentInput"; import { getProcessVersionId } from "../../reducers/selectors/graph"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; interface CustomActionFormProps extends ChangeableValue { action: CustomAction; @@ -125,32 +126,25 @@ export function CustomActionDialog(props: WindowContentProps [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.confirm", "Ok"), action: () => confirmAction() }, + { title: t("dialog.button.confirm", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); return ( - +
- + setComment(e.target.value)} - value={comment} - defaultValue={deploymentCommentSettings?.exampleComment} - className={cx( - css({ - minWidth: 600, - minHeight: 80, - }), - )} autoFocus /> - - {validationError} -
- + ); } diff --git a/designer/client/src/components/modals/DeployProcessDialog.tsx b/designer/client/src/components/modals/DeployProcessDialog.tsx index 369209ca610..2eae9482915 100644 --- a/designer/client/src/components/modals/DeployProcessDialog.tsx +++ b/designer/client/src/components/modals/DeployProcessDialog.tsx @@ -3,26 +3,48 @@ import { WindowButtonProps, WindowContentProps } from "@touk/window-manager"; import React, { useCallback, useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; -import { getProcessName } from "../../reducers/selectors/graph"; +import { getActivityParameters, getProcessName } from "../../reducers/selectors/graph"; import { getFeatureSettings } from "../../reducers/selectors/settings"; import { ProcessName } from "../Process/types"; import { PromptContent, WindowKind } from "../../windowManager"; -import CommentInput from "../comment/CommentInput"; -import ProcessDialogWarnings from "./ProcessDialogWarnings"; -import { FormHelperText, Typography } from "@mui/material"; import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; +import { ActivityNodeParameters } from "../../types/activity"; +import { AdvancedParametersSection } from "./AdvancedParametersSection"; +import { mapValues } from "lodash"; +import { NodesDeploymentData } from "../../http/HttpService"; +import { ActivityProperty } from "./ActivityProperty"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; +import { NodeTable } from "../graph/node-modal/NodeDetailsContent/NodeTable"; export type ToggleProcessActionModalData = { - action: (processName: ProcessName, comment: string) => Promise; + action: (processName: ProcessName, comment: string, nodeData: NodesDeploymentData) => Promise; + activityName: string; displayWarnings?: boolean; }; +function initialNodesData(params: ActivityNodeParameters[]) { + return params.reduce( + (paramObj, { nodeId, parameters }) => ({ + ...paramObj, + [nodeId]: mapValues(parameters, (value) => value.defaultValue || ""), + }), + {}, + ); +} + export function DeployProcessDialog(props: WindowContentProps): JSX.Element { // TODO: get rid of meta const { - meta: { action, displayWarnings }, + meta: { action, activityName, displayWarnings }, } = props.data; const processName = useSelector(getProcessName); + + const activityParameters = useSelector(getActivityParameters); + const activityNodeParameters = activityParameters[activityName] || ([] as ActivityNodeParameters[]); + const initialValues = useMemo(() => initialNodesData(activityNodeParameters), [activityNodeParameters]); + const [values, setValues] = useState(initialValues); + const [comment, setComment] = useState(""); const [validationError, setValidationError] = useState(""); const featureSettings = useSelector(getFeatureSettings); @@ -32,7 +54,7 @@ export function DeployProcessDialog(props: WindowContentProps { try { - await action(processName, comment); + await action(processName, comment, values); props.close(); } catch (error) { setValidationError(error?.response?.data); @@ -42,32 +64,54 @@ export function DeployProcessDialog(props: WindowContentProps [ - { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), action: () => confirmAction() }, + { + title: t("dialog.button.cancel", "Cancel"), + action: () => props.close(), + classname: LoadingButtonTypes.secondaryButton, + }, + { title: t("dialog.button.ok", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); return ( -
- {props.data.title} - {displayWarnings && } - + + setComment(e.target.value)} - value={comment} - defaultValue={deploymentCommentSettings?.exampleComment} - className={cx( - css({ - minWidth: 600, - minHeight: 80, - }), - )} autoFocus /> - - {validationError} - + {activityNodeParameters.map((anp: ActivityNodeParameters) => ( + + + {Object.entries(anp.parameters).map(([paramName, paramConfig]) => { + return ( + { + setValues({ + ...values, + [nodeId]: { + ...values[nodeId], + [paramName]: newValue, + }, + }); + }} + nodesData={values} + /> + ); + })} + + + ))}
); diff --git a/designer/client/src/components/modals/GenerateTestDataDialog.tsx b/designer/client/src/components/modals/GenerateTestDataDialog.tsx index 08189d2120e..764a8c30d50 100644 --- a/designer/client/src/components/modals/GenerateTestDataDialog.tsx +++ b/designer/client/src/components/modals/GenerateTestDataDialog.tsx @@ -46,7 +46,7 @@ function GenerateTestDataDialog(props: WindowContentProps): JSX.Element { const buttons: WindowButtonProps[] = useMemo( () => [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), disabled: !isValid, action: () => confirmAction() }, + { title: t("dialog.button.ok", "Apply"), disabled: !isValid, action: () => confirmAction() }, ], [t, confirmAction, props, isValid], ); diff --git a/designer/client/src/components/modals/SaveProcessDialog.tsx b/designer/client/src/components/modals/SaveProcessDialog.tsx index de9299dd49e..1a30c17190d 100644 --- a/designer/client/src/components/modals/SaveProcessDialog.tsx +++ b/designer/client/src/components/modals/SaveProcessDialog.tsx @@ -5,7 +5,6 @@ import { useTranslation } from "react-i18next"; import { useDispatch } from "react-redux"; import { displayCurrentProcessVersion, loadProcessToolbarsConfiguration } from "../../actions/nk"; import { PromptContent } from "../../windowManager"; -import { CommentInput } from "../comment/CommentInput"; import { ThunkAction } from "../../actions/reduxTypes"; import { getScenarioGraph, @@ -18,9 +17,10 @@ import HttpService from "../../http/HttpService"; import { ActionCreators as UndoActionCreators } from "redux-undo"; import { visualizationUrl } from "../../common/VisualizationUrl"; import { useLocation, useNavigate } from "react-router-dom"; -import { Typography } from "@mui/material"; import { LoadingButtonTypes } from "../../windowManager/LoadingButton"; import { getScenarioActivities } from "../../actions/nk/scenarioActivities"; +import { ActivityCommentTextField } from "./ActivityCommentTextField"; +import { ActivityHeader } from "./ActivityHeader"; export function SaveProcessDialog(props: WindowContentProps): JSX.Element { const location = useLocation(); @@ -71,7 +71,7 @@ export function SaveProcessDialog(props: WindowContentProps): JSX.Element { const buttons: WindowButtonProps[] = useMemo( () => [ { title: t("dialog.button.cancel", "Cancel"), action: () => props.close(), classname: LoadingButtonTypes.secondaryButton }, - { title: t("dialog.button.ok", "Ok"), action: () => confirmAction() }, + { title: t("dialog.button.ok", "Apply"), action: () => confirmAction() }, ], [confirmAction, props, t], ); @@ -79,16 +79,8 @@ export function SaveProcessDialog(props: WindowContentProps): JSX.Element { return (
- {props.data.title} - setState(e.target.value)} - value={comment} - className={css({ - minWidth: 600, - minHeight: 80, - })} - autoFocus - /> + + setState(e.target.value)} autoFocus />
); diff --git a/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx b/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx index 44e25d994ee..19812e0277a 100644 --- a/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx +++ b/designer/client/src/components/toolbars/process/buttons/SaveButton.tsx @@ -17,8 +17,8 @@ function SaveButton(props: ToolbarButtonProps): JSX.Element { const unsavedNewName = useSelector(getProcessUnsavedNewName); const isRenamed = useSelector(isProcessRenamed); const title = isRenamed - ? t("saveProcess.renameTitle", "Save scenario as {{name}}", { name: unsavedNewName }) - : t("saveProcess.title", "Save scenario {{name}}", { name: processName }); + ? t("saveProcess.renameTitle", "Save scenario as", { name: unsavedNewName }) + : t("saveProcess.title", "Save scenario", { name: processName }); const { open } = useWindows(); const onClick = () => diff --git a/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx b/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx index be1c506afa1..cb6a24f891d 100644 --- a/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx +++ b/designer/client/src/components/toolbars/scenarioActions/buttons/CancelDeployButton.tsx @@ -36,7 +36,7 @@ export default function CancelDeployButton(props: ToolbarButtonProps) { title: message, kind: WindowKind.deployProcess, width: ACTION_DIALOG_WIDTH, - meta: { action }, + meta: { action, activityName: "CANCEL" }, }) } type={type} diff --git a/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx b/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx index ab685cbe134..cd424cb5736 100644 --- a/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx +++ b/designer/client/src/components/toolbars/scenarioActions/buttons/DeployButton.tsx @@ -13,6 +13,8 @@ import { ToolbarButton } from "../../../toolbarComponents/toolbarButtons"; import { ToolbarButtonProps } from "../../types"; import { ACTION_DIALOG_WIDTH } from "../../../../stylesheets/variables"; +import { useActivityCapabilities } from "../../../modals/GenericAction/useActivityCapabilities"; + export default function DeployButton(props: ToolbarButtonProps) { const dispatch = useDispatch(); const deployPossible = useSelector(isDeployPossible); @@ -23,6 +25,9 @@ export default function DeployButton(props: ToolbarButtonProps) { const capabilities = useSelector(getCapabilities); const { disabled, type } = props; + // TODO: find better place to reload activity capabilities and properties + useActivityCapabilities(); + const available = !disabled && deployPossible && capabilities.deploy; const { t } = useTranslation(); @@ -39,7 +44,7 @@ export default function DeployButton(props: ToolbarButtonProps) { const { open } = useWindows(); const message = t("panels.actions.deploy.dialog", "Deploy scenario {{name}}", { name: processName }); - const action = (p, c) => HttpService.deploy(p, c).finally(() => dispatch(loadProcessState(processName, processVersionId))); + const action = (p, c, d) => HttpService.deploy(p, c, d).finally(() => dispatch(loadProcessState(processName, processVersionId))); return ( >; + export type NodeUsageData = { fragmentNodeId?: string; nodeId: string; @@ -337,11 +339,16 @@ class HttpService { deploy( processName: string, comment?: string, + nodesDeploymentData?: NodesDeploymentData, ): Promise<{ isSuccess: boolean; }> { + const runDeploymentRequest = { + ...(nodesDeploymentData && { nodesDeploymentData: nodesDeploymentData }), + ...(comment && { comment: comment }), + }; return api - .post(`/processManagement/deploy/${encodeURIComponent(processName)}`, comment) + .post(`/processManagement/deploy/${encodeURIComponent(processName)}`, runDeploymentRequest) .then(() => { return { isSuccess: true }; }) @@ -700,6 +707,21 @@ class HttpService { return promise; } + getActivityParameters(processName: string, scenarioGraph: ScenarioGraph) { + const promise = api.post( + `/activityInfo/${encodeURIComponent(processName)}/activityParameters`, + this.#sanitizeScenarioGraph(scenarioGraph), + ); + promise.catch((error) => + this.#addError( + i18next.t("notification.error.failedToGetTestParameters", "Failed to get activity parameters definition"), + error, + true, + ), + ); + return promise; + } + generateTestData(processName: string, testSampleSize: string, scenarioGraph: ScenarioGraph): Promise { const promise = api.post( `/scenarioTesting/${encodeURIComponent(processName)}/generate/${testSampleSize}`, diff --git a/designer/client/src/reducers/graph/reducer.ts b/designer/client/src/reducers/graph/reducer.ts index b5bbaacafd9..bc69852ebba 100644 --- a/designer/client/src/reducers/graph/reducer.ts +++ b/designer/client/src/reducers/graph/reducer.ts @@ -84,6 +84,12 @@ const graphReducer: Reducer = (state = emptyGraphState, action) => { testFormParameters: action.testFormParameters, }; } + case "UPDATE_ACTIVITY_PARAMETERS": { + return { + ...state, + activityParameters: action.activityParameters, + }; + } case "DISPLAY_PROCESS": { const { scenario } = action; return { diff --git a/designer/client/src/reducers/graph/types.ts b/designer/client/src/reducers/graph/types.ts index 4f4002057f5..5e16836f8c0 100644 --- a/designer/client/src/reducers/graph/types.ts +++ b/designer/client/src/reducers/graph/types.ts @@ -1,6 +1,7 @@ import { Layout, RefreshData } from "../../actions/nk"; import { Scenario } from "../../components/Process/types"; import { TestCapabilities, TestFormParameters, TestResults } from "../../common/TestResultUtils"; +import { ActivityParameters } from "../../types/activity"; export interface NodeCounts { errors?: number; @@ -15,6 +16,7 @@ export type GraphState = { scenario?: Scenario; selectionState?: string[]; layout: Layout; + activityParameters?: ActivityParameters; testCapabilities?: TestCapabilities; testFormParameters?: TestFormParameters[]; testResults: TestResults; diff --git a/designer/client/src/reducers/selectors/graph.ts b/designer/client/src/reducers/selectors/graph.ts index 3fbf656d70a..8a3b2cdd428 100644 --- a/designer/client/src/reducers/selectors/graph.ts +++ b/designer/client/src/reducers/selectors/graph.ts @@ -8,6 +8,7 @@ import { ProcessCounts } from "../graph"; import { RootState } from "../index"; import { getProcessState } from "./scenarioState"; import { TestFormParameters } from "../../common/TestResultUtils"; +import { ActivityParameters } from "../../types/activity"; export const getGraph = (state: RootState) => state.graphReducer.history.present; @@ -70,6 +71,7 @@ export const isArchivePossible = createSelector( [getProcessState, isFragment], (state, isFragment) => isFragment || ProcessStateUtils.canArchive(state), ); +export const getActivityParameters = createSelector(getGraph, (g) => g.activityParameters || ({} as ActivityParameters)); export const getTestCapabilities = createSelector(getGraph, (g) => g.testCapabilities); export const getTestParameters = createSelector(getGraph, (g) => g.testFormParameters || ([] as TestFormParameters[])); export const getTestResults = createSelector(getGraph, (g) => g.testResults); diff --git a/designer/client/src/types/activity.ts b/designer/client/src/types/activity.ts new file mode 100644 index 00000000000..636b095b786 --- /dev/null +++ b/designer/client/src/types/activity.ts @@ -0,0 +1,19 @@ +import { NodeId } from "./node"; + +export interface ActivityParameterConfig { + editor: any; + label: string; + defaultValue: string | null; + hintText: string | null; +} + +export type ActivityParameterName = string; + +export interface ActivityNodeParameters { + nodeId: NodeId; + parameters: { [key: ActivityParameterName]: ActivityParameterConfig }; +} + +export type ActivityName = string; + +export type ActivityParameters = { [key: ActivityName]: ActivityNodeParameters[] }; diff --git a/designer/client/src/windowManager/PromptContent.tsx b/designer/client/src/windowManager/PromptContent.tsx index 37a76f38740..511f5adfc08 100644 --- a/designer/client/src/windowManager/PromptContent.tsx +++ b/designer/client/src/windowManager/PromptContent.tsx @@ -16,7 +16,14 @@ export function PromptContent(props: PropsWithChildren): JS paddingLeft: theme.custom.spacing.baseUnit * 6, paddingRight: theme.custom.spacing.baseUnit * 6, }); - return { ...props.classnames, content }; + return { + footer: css({ + justifyContent: "flex-end", + backgroundColor: "#1D2734", + }), + ...props.classnames, + content, + }; }, [props.classnames, theme.custom.spacing.baseUnit]); const components = useMemo( diff --git a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala index 11dcd56935f..37a72fa6fdb 100644 --- a/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala +++ b/designer/restmodel/src/main/scala/pl/touk/nussknacker/restmodel/definition/package.scala @@ -136,6 +136,13 @@ package object definition { hintText: Option[String] ) + @JsonCodec final case class UiActivityParameterConfig( + defaultValue: Option[String], + editor: ParameterEditor, + label: Option[String], + hintText: Option[String] + ) + object UIParameter { implicit def decoder(implicit typing: Decoder[TypingResult]): Decoder[UIParameter] = deriveConfiguredDecoder[UIParameter] diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala new file mode 100644 index 00000000000..aee3cec6b4d --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ActivityInfoResources.scala @@ -0,0 +1,45 @@ +package pl.touk.nussknacker.ui.api + +import akka.http.scaladsl.server.{Directives, Route} +import com.typesafe.scalalogging.LazyLogging +import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.ui.api.utils.ScenarioDetailsOps.ScenarioWithDetailsOps +import pl.touk.nussknacker.ui.process.ProcessService +import pl.touk.nussknacker.ui.process.newactivity.ActivityInfoService +import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider +import pl.touk.nussknacker.ui.security.api.LoggedUser + +import scala.concurrent.ExecutionContext + +class ActivityInfoResources( + protected val processService: ProcessService, + activityInfoService: ProcessingTypeDataProvider[ActivityInfoService, _] +)(implicit val ec: ExecutionContext) + extends Directives + with FailFastCirceSupport + with RouteWithUser + with ProcessDirectives + with LazyLogging { + + def securedRoute(implicit user: LoggedUser): Route = { + pathPrefix("activityInfo" / ProcessNameSegment) { processName => + (post & processDetailsForName(processName)) { processDetails => + entity(as[ScenarioGraph]) { scenarioGraph => + path("activityParameters") { + complete { + activityInfoService + .forProcessingTypeUnsafe(processDetails.processingType) + .getActivityParameters( + scenarioGraph, + processDetails.processVersionUnsafe, + processDetails.isFragment + ) + } + } + } + } + } + } + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala index 19d64caba1f..11bf71667e7 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/DeploymentApiHttpService.scala @@ -1,6 +1,7 @@ package pl.touk.nussknacker.ui.api import pl.touk.nussknacker.engine.api.Comment +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.ProblemDeploymentStatus import pl.touk.nussknacker.ui.api.description.DeploymentApiEndpoints import pl.touk.nussknacker.ui.api.description.DeploymentApiEndpoints.Dtos._ @@ -30,7 +31,9 @@ class DeploymentApiHttpService( RunDeploymentCommand( id = deploymentId, scenarioName = request.scenarioName, - nodesDeploymentData = request.nodesDeploymentData, + nodesDeploymentData = NodesDeploymentData(request.nodesDeploymentData.map { case (nodeId, paramValue) => + (nodeId, Map("sqlExpression" -> paramValue)) + }), user = loggedUser ), request.comment.flatMap(Comment.from) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala index 37964bc3a96..d6ab6658147 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiHttpService.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui.services +package pl.touk.nussknacker.ui.api import cats.data.{EitherT, Validated} import cats.syntax.all._ @@ -9,9 +9,13 @@ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, Pro import pl.touk.nussknacker.engine.deployment.CustomActionDefinition import pl.touk.nussknacker.restmodel.CustomActionRequest import pl.touk.nussknacker.restmodel.validation.PrettyValidationErrors -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError.{NoActionDefinition, NoScenario} -import pl.touk.nussknacker.ui.api.{BaseHttpService, CustomActionValidationDto, ManagementApiEndpoints} +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints.Dtos.{ + CustomActionValidationDto, + ManagementApiError, + NoActionDefinition, + NoScenario +} import pl.touk.nussknacker.ui.process.ProcessService import pl.touk.nussknacker.ui.process.deployment.DeploymentManagerDispatcher import pl.touk.nussknacker.ui.security.api.{AuthManager, LoggedUser} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala index 89a35068ceb..012e1ccb6b4 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementResources.scala @@ -6,6 +6,7 @@ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} import com.typesafe.scalalogging.LazyLogging import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport +import io.circe.generic.JsonCodec import io.circe.generic.extras.semiauto.deriveConfiguredEncoder import io.circe.{Decoder, Encoder, Json, parser} import io.dropwizard.metrics5.MetricRegistry @@ -66,6 +67,11 @@ object ManagementResources { } + @JsonCodec final case class RunDeploymentRequest( + nodesDeploymentData: Option[NodesDeploymentData], + comment: Option[String] + ) + } class ManagementResources( @@ -119,16 +125,16 @@ class ManagementResources( } } ~ path("deploy" / ProcessNameSegment) { processName => - (post & processId(processName) & entity(as[Option[String]]) & parameters(Symbol("savepointPath"))) { - (processIdWithName, comment, savepointPath) => + (post & processId(processName) & entity(as[RunDeploymentRequest]) & parameters(Symbol("savepointPath"))) { + (processIdWithName, request, savepointPath) => canDeploy(processIdWithName) { complete { deploymentService .processCommand( RunDeploymentCommand( // adminProcessManagement endpoint is not used by the designer client. It is a part of API for tooling purpose - commonData = CommonCommandData(processIdWithName, comment.flatMap(Comment.from), user), - nodesDeploymentData = NodesDeploymentData.empty, + commonData = CommonCommandData(processIdWithName, request.comment.flatMap(Comment.from), user), + nodesDeploymentData = request.nodesDeploymentData.getOrElse(NodesDeploymentData.empty), stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromCustomSavepoint(savepointPath) ) ) @@ -142,15 +148,15 @@ class ManagementResources( pathPrefix("processManagement") { path("deploy" / ProcessNameSegment) { processName => - (post & processId(processName) & entity(as[Option[String]])) { (processIdWithName, comment) => + (post & processId(processName) & entity(as[RunDeploymentRequest])) { (processIdWithName, request) => canDeploy(processIdWithName) { complete { measureTime("deployment", metricRegistry) { deploymentService .processCommand( RunDeploymentCommand( - commonData = CommonCommandData(processIdWithName, comment.flatMap(Comment.from), user), - nodesDeploymentData = NodesDeploymentData.empty, + commonData = CommonCommandData(processIdWithName, request.comment.flatMap(Comment.from), user), + nodesDeploymentData = request.nodesDeploymentData.getOrElse(NodesDeploymentData.empty), stateRestoringStrategy = StateRestoringStrategy.RestoreStateFromReplacedJobSavepoint ) ) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala index c733cb4682d..759835556b5 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/DeploymentApiEndpoints.scala @@ -4,7 +4,6 @@ import cats.data.NonEmptyList import derevo.circe.{decoder, encoder} import derevo.derive import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.{NodeDeploymentData, NodesDeploymentData, SqlFilteringExpression} import pl.touk.nussknacker.engine.api.context.ProcessCompilationError.{ EmptyProcess, ExpressionParserCompilationError, @@ -48,9 +47,7 @@ class DeploymentApiEndpoints(auth: EndpointInput[AuthCredentials]) extends BaseE .example( RunDeploymentRequest( scenarioName = ProcessName("scenario1"), - NodesDeploymentData( - Map(NodeId("sourceNodeId1") -> SqlFilteringExpression("field1 = 'value'")) - ), + nodesDeploymentData = Map(NodeId("sourceNodeId1") -> "field1 = 'value'"), comment = None ) ) @@ -200,7 +197,7 @@ object DeploymentApiEndpoints { @derive(encoder, decoder, schema) final case class RunDeploymentRequest( scenarioName: ProcessName, - nodesDeploymentData: NodesDeploymentData, + nodesDeploymentData: Map[NodeId, String], // nodeId -> single parameter value (currently sqlExpression) // NodesDeploymentData, comment: Option[String] ) @@ -213,13 +210,7 @@ object DeploymentApiEndpoints { modifiedAt: Instant ) - implicit val nodeDeploymentDataCodec: Schema[NodeDeploymentData] = Schema.string[SqlFilteringExpression].as - - implicit val nodesDeploymentDataCodec: Schema[NodesDeploymentData] = Schema - .schemaForMap[NodeId, NodeDeploymentData](_.id) - .map[NodesDeploymentData]((map: Map[NodeId, NodeDeploymentData]) => Some(NodesDeploymentData(map)))( - _.dataByNodeId - ) + implicit val nodesDeploymentDataCodec: Schema[Map[NodeId, String]] = Schema.schemaForMap[NodeId, String](_.id) sealed trait RunDeploymentError diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala similarity index 87% rename from designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala index a64a65b4812..c168561e1de 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/ManagementApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/ManagementApiEndpoints.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.ui.api +package pl.touk.nussknacker.ui.api.description import derevo.circe.encoder import derevo.derive @@ -9,8 +9,7 @@ import pl.touk.nussknacker.restmodel.BaseEndpointDefinitions.SecuredEndpoint import pl.touk.nussknacker.restmodel.validation.ValidationResults.NodeValidationError import pl.touk.nussknacker.restmodel.{BaseEndpointDefinitions, CustomActionRequest} import pl.touk.nussknacker.security.AuthCredentials -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError -import pl.touk.nussknacker.ui.api.ManagementApiEndpoints.ManagementApiError.{NoActionDefinition, NoScenario} +import pl.touk.nussknacker.ui.api.description.ManagementApiEndpoints.Dtos.{CustomActionValidationDto, ManagementApiError, NoActionDefinition, NoScenario} import pl.touk.nussknacker.ui.api.TapirCodecs.ScenarioNameCodec._ import pl.touk.nussknacker.ui.api.TapirCodecs.ClassCodec._ import pl.touk.nussknacker.ui.api.BaseHttpService.CustomAuthorizationError @@ -59,14 +58,15 @@ class ManagementApiEndpoints(auth: EndpointInput[AuthCredentials]) extends BaseE } -@derive(schema, encoder) -final case class CustomActionValidationDto(validationErrors: List[NodeValidationError], validationPerformed: Boolean) object ManagementApiEndpoints { - sealed trait ManagementApiError + object Dtos { - object ManagementApiError { + @derive(schema, encoder) + final case class CustomActionValidationDto(validationErrors: List[NodeValidationError], validationPerformed: Boolean) + + sealed trait ManagementApiError final case object NoPermission extends ManagementApiError with CustomAuthorizationError final case class NoScenario(scenarioName: ProcessName) extends ManagementApiError final case class NoActionDefinition(scenarioName: ProcessName, actionName: ScenarioActionName) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala index cedbe843136..19d7251210c 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/api/description/NodesApiEndpoints.scala @@ -17,7 +17,7 @@ import pl.touk.nussknacker.engine.api.definition.{ ParameterEditor, SimpleParameterEditor } -import pl.touk.nussknacker.engine.api.editor.DualEditorMode +import pl.touk.nussknacker.engine.api.editor.{DualEditorMode, FixedValuesEditorMode} import pl.touk.nussknacker.engine.api.generics.ExpressionParseError.{CellError, ColumnDefinition, ErrorDetails} import pl.touk.nussknacker.engine.api.graph.{Edge, ProcessProperties, ScenarioGraph} import pl.touk.nussknacker.engine.api.parameter.{ @@ -1304,6 +1304,7 @@ object NodesApiEndpoints { implicit lazy val simpleParameterEditorSchema: Schema[SimpleParameterEditor] = Schema.derived implicit lazy val parameterEditorSchema: Schema[ParameterEditor] = Schema.derived implicit lazy val dualEditorSchema: Schema[DualEditorMode] = Schema.string + implicit lazy val fixedValuesEditorMode: Schema[FixedValuesEditorMode] = Schema.string implicit lazy val durationSchema: Schema[Duration] = Schema.schemaForJavaDuration implicit lazy val uiParameterSchema: Schema[UIParameter] = Schema.derived diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala new file mode 100644 index 00000000000..74d59891cd9 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/newactivity/ActivityInfoService.scala @@ -0,0 +1,59 @@ +package pl.touk.nussknacker.ui.process.newactivity + +import io.circe.generic.JsonCodec +import pl.touk.nussknacker.engine.api.NodeId +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.definition.RawParameterEditor +import pl.touk.nussknacker.engine.api.graph.ScenarioGraph +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.definition.activity.ActivityInfoProvider +import pl.touk.nussknacker.restmodel.definition.UiActivityParameterConfig +import pl.touk.nussknacker.ui.process.newactivity.ActivityInfoService.{ActivityName, UiActivityNodeParameters} +import pl.touk.nussknacker.ui.security.api.LoggedUser +import pl.touk.nussknacker.ui.uiresolving.UIProcessResolver + +// TODO: move to ActivityService? execute node compilation only once with ScenarioTestService? +class ActivityInfoService(activityInfoProvider: ActivityInfoProvider, processResolver: UIProcessResolver) { + + def getActivityParameters( + scenarioGraph: ScenarioGraph, + processVersion: ProcessVersion, + isFragment: Boolean + )( + implicit user: LoggedUser + ): Map[ActivityName, List[UiActivityNodeParameters]] = { + val canonical = toCanonicalProcess(scenarioGraph, processVersion, isFragment) + activityInfoProvider + .getActivityParameters(processVersion, canonical) + .map { case (activityName, nodeParamsMap) => + activityName -> nodeParamsMap.map { case (nodeId, params) => + UiActivityNodeParameters( + NodeId(nodeId), + params.map { case (name, value) => + name -> UiActivityParameterConfig( + value.defaultValue, + value.editor.getOrElse(RawParameterEditor), + value.label, + value.hintText + ) + } + ) + }.toList + } + } + + // copied from ScenarioTestService + private def toCanonicalProcess( + scenarioGraph: ScenarioGraph, + processVersion: ProcessVersion, + isFragment: Boolean + )(implicit user: LoggedUser): CanonicalProcess = { + processResolver.validateAndResolve(scenarioGraph, processVersion, isFragment) + } + +} + +object ActivityInfoService { + type ActivityName = String + @JsonCodec case class UiActivityNodeParameters(nodeId: NodeId, parameters: Map[String, UiActivityParameterConfig]) +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index bb1356a2319..b6210e79380 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -13,6 +13,7 @@ import pl.touk.nussknacker.engine.api.component._ import pl.touk.nussknacker.engine.api.process.ProcessingType import pl.touk.nussknacker.engine.compile.ProcessValidator import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode +import pl.touk.nussknacker.engine.definition.activity.ModelDataActivityInfoProvider import pl.touk.nussknacker.engine.definition.test.ModelDataTestInfoProvider import pl.touk.nussknacker.engine.dict.ProcessDictSubstitutor import pl.touk.nussknacker.engine.util.loader.ScalaServiceLoader @@ -57,7 +58,7 @@ import pl.touk.nussknacker.ui.process.deployment.{ import pl.touk.nussknacker.ui.process.fragment.{DefaultFragmentRepository, FragmentResolver} import pl.touk.nussknacker.ui.process.label.ScenarioLabelsService import pl.touk.nussknacker.ui.process.migrate.{HttpRemoteEnvironment, ProcessModelMigrator, TestModelMigrations} -import pl.touk.nussknacker.ui.process.newactivity.ActivityService +import pl.touk.nussknacker.ui.process.newactivity.{ActivityInfoService, ActivityService} import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizationConfig, DeploymentsStatusesSynchronizationScheduler, @@ -74,7 +75,7 @@ import pl.touk.nussknacker.ui.process.test.{PreliminaryScenarioTestDataSerDe, Sc import pl.touk.nussknacker.ui.process.version.{ScenarioGraphVersionRepository, ScenarioGraphVersionService} import pl.touk.nussknacker.ui.processreport.ProcessCounter import pl.touk.nussknacker.ui.security.api.{AuthManager, AuthenticationResources} -import pl.touk.nussknacker.ui.services.{ManagementApiHttpService, NuDesignerExposedApiHttpService} +import pl.touk.nussknacker.ui.services.NuDesignerExposedApiHttpService import pl.touk.nussknacker.ui.statistics.repository.FingerprintRepositoryImpl import pl.touk.nussknacker.ui.statistics.{ FingerprintService, @@ -215,6 +216,12 @@ class AkkaHttpBasedRouteProvider( new ScenarioTestExecutorServiceImpl(scenarioResolver, deploymentManager) ) } + val scenarioActivityService = scenarioTestServiceDeps.mapValues { case (_, processResolver, _, modelData, _) => + new ActivityInfoService( + new ModelDataActivityInfoProvider(modelData), + processResolver + ) + } val processValidator = scenarioTestServiceDeps.mapValues(_._1) val processResolver = scenarioTestServiceDeps.mapValues(_._2) @@ -510,6 +517,7 @@ class AkkaHttpBasedRouteProvider( ) } ), + new ActivityInfoResources(processService, scenarioActivityService), new StatusResources(stateDefinitionService), ) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala index cde22f36654..8dbfd9bb49a 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/it/NuResourcesTest.scala @@ -39,6 +39,7 @@ import pl.touk.nussknacker.test.utils.domain.TestFactory._ import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} import pl.touk.nussknacker.test.utils.scalas.AkkaHttpExtensions.toRequestEntity import pl.touk.nussknacker.ui.LoadableConfigBasedNussknackerConfig +import pl.touk.nussknacker.ui.api.ManagementResources.RunDeploymentRequest import pl.touk.nussknacker.ui.api._ import pl.touk.nussknacker.ui.config.FeatureTogglesConfig import pl.touk.nussknacker.ui.config.scenariotoolbar.CategoriesScenarioToolbarsConfigParser @@ -336,7 +337,10 @@ trait NuResourcesTest ): RouteTestResult = Post( s"/processManagement/deploy/$processName", - HttpEntity(ContentTypes.`application/json`, comment.getOrElse("")) + HttpEntity( + ContentTypes.`application/json`, + RunDeploymentRequest(None, comment).asJson.noSpaces + ) ) ~> withPermissions(deployRoute(), Permission.Deploy, Permission.Read) diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala new file mode 100644 index 00000000000..1bfc22968d2 --- /dev/null +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ActivityInfoResourcesSpec.scala @@ -0,0 +1,88 @@ +package pl.touk.nussknacker.ui.api + +import io.restassured.RestAssured.`given` +import io.restassured.module.scala.RestAssuredSupport.AddThenToResponse +import org.hamcrest.Matchers.{equalTo, notNullValue} +import org.scalatest.freespec.AnyFreeSpecLike +import pl.touk.nussknacker.engine.build.ScenarioBuilder +import pl.touk.nussknacker.test.base.it.{NuItTest, WithSimplifiedConfigScenarioHelper} +import pl.touk.nussknacker.test.config.{WithBusinessCaseRestAssuredUsersExtensions, WithSimplifiedDesignerConfig} +import pl.touk.nussknacker.test.{NuRestAssureMatchers, RestAssuredVerboseLoggingIfValidationFails} +import pl.touk.nussknacker.engine.spel.SpelExtension._ +import pl.touk.nussknacker.test.utils.domain.TestProcessUtil + +class ActivityInfoResourcesSpec + extends AnyFreeSpecLike + with NuItTest + with WithSimplifiedDesignerConfig + with WithSimplifiedConfigScenarioHelper + with WithBusinessCaseRestAssuredUsersExtensions + with NuRestAssureMatchers + with RestAssuredVerboseLoggingIfValidationFails { + + "The scenario activity info endpoint when" - { + "return activity parameters when defined" in { + val scenario = ScenarioBuilder + .streaming("scenarioWithSourceWithDeployParameters") + .source("sourceWithParametersId", "boundedSourceWithOffset", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .applicationState { + createSavedScenario(scenario) + } + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(200) + .body( + "DEPLOY[0].nodeId", + equalTo("sourceWithParametersId"), + "DEPLOY[0].parameters.offset", + notNullValue(), + ) + } + + "return empty map when no activity parameters" in { + val scenario = ScenarioBuilder + .streaming("scenarioWithoutParameters") + .source("sourceNoParamsId", "boundedSource", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .applicationState { + createSavedScenario(scenario) + } + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(200) + .equalsJsonBody( + "{}" + ) + } + + "return no data found when there is no scenario" in { + val scenario = ScenarioBuilder + .streaming("invalidScenario") + .source("exampleSource", "boundedSource", "elements" -> "{'one', 'two', 'three'}".spel) + .emptySink("exampleSinkId", "emptySink") + + given() + .when() + .basicAuthAllPermUser() + .jsonBody(TestProcessUtil.toJson(scenario).noSpaces) + .post(s"$nuDesignerHttpAddress/api/activityInfo/${scenario.name.value}/activityParameters") + .Then() + .statusCode(404) + .equalsPlainBody( + s"No scenario ${scenario.name.value} found" + ) + } + } + +} diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala index 449d2f851a2..549c7ca717f 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceBusinessSpec.scala @@ -178,6 +178,7 @@ class ComponentApiHttpServiceBusinessSpec "streaming-sink-monitor", "streaming-sink-sendsms", "streaming-source-boundedsource", + "streaming-source-boundedsourcewithoffset", "streaming-source-classinstancesource", "streaming-source-communicationsource", "streaming-source-csv-source", diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala index 218897b3200..efbe512da75 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ComponentApiHttpServiceSecuritySpec.scala @@ -262,6 +262,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming1-sink-monitor", "streaming1-sink-sendsms", "streaming1-source-boundedsource", + "streaming1-source-boundedsourcewithoffset", "streaming1-source-classinstancesource", "streaming1-source-communicationsource", "streaming1-source-csv-source", @@ -327,6 +328,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming2-sink-monitor", "streaming2-sink-sendsms", "streaming2-source-boundedsource", + "streaming2-source-boundedsourcewithoffset", "streaming2-source-classinstancesource", "streaming2-source-communicationsource", "streaming2-source-csv-source", @@ -391,6 +393,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming1-sink-monitor", "streaming1-sink-sendsms", "streaming1-source-boundedsource", + "streaming1-source-boundedsourcewithoffset", "streaming1-source-classinstancesource", "streaming1-source-communicationsource", "streaming1-source-csv-source", @@ -448,6 +451,7 @@ class ComponentApiHttpServiceSecuritySpec "streaming2-sink-monitor", "streaming2-sink-sendsms", "streaming2-source-boundedsource", + "streaming2-source-boundedsourcewithoffset", "streaming2-source-classinstancesource", "streaming2-source-communicationsource", "streaming2-source-csv-source", diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DefinitionResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DefinitionResourcesSpec.scala index e79e236834e..d5728d0d6b6 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DefinitionResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/DefinitionResourcesSpec.scala @@ -153,13 +153,16 @@ class DefinitionResourcesSpec .parse("""{"possibleValues" : [ | { | "expression" : "", - | "label" : "" + | "label" : "", + | "hintText" : null | }, | { | "expression" : "'someValue'", - | "label" : "someValue" + | "label" : "someValue", + | "hintText" : null | } | ], + | "mode" : "LIST", | "type" : "FixedValuesParameterEditor" |}""".stripMargin) .toOption diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala index 988b54345e8..ec88edbcc7e 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/ManagementResourcesSpec.scala @@ -1,6 +1,6 @@ package pl.touk.nussknacker.ui.api -import akka.http.scaladsl.model.{ContentTypeRange, StatusCodes} +import akka.http.scaladsl.model.{ContentTypeRange, ContentTypes, HttpEntity, StatusCodes} import akka.http.scaladsl.server import akka.http.scaladsl.testkit.ScalatestRouteTest import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} @@ -26,6 +26,7 @@ import pl.touk.nussknacker.test.base.it.NuResourcesTest import pl.touk.nussknacker.test.mock.MockDeploymentManager import pl.touk.nussknacker.test.utils.domain.TestFactory.{withAllPermissions, withPermissions} import pl.touk.nussknacker.test.utils.domain.{ProcessTestData, TestFactory} +import pl.touk.nussknacker.ui.api.ManagementResources.RunDeploymentRequest import pl.touk.nussknacker.ui.api.description.scenarioActivity.Dtos import pl.touk.nussknacker.ui.process.ScenarioQuery import pl.touk.nussknacker.ui.process.exception.ProcessIllegalAction @@ -234,8 +235,15 @@ class ManagementResourcesSpec } test("not authorize user with write permission to deploy") { + import io.circe.syntax._ saveCanonicalProcessAndAssertSuccess(ProcessTestData.sampleScenario) - Post(s"/processManagement/deploy/${ProcessTestData.sampleScenario.name}") ~> withPermissions( + Post( + s"/processManagement/deploy/${ProcessTestData.sampleScenario.name}", + HttpEntity( + ContentTypes.`application/json`, + RunDeploymentRequest(None, None).asJson.noSpaces + ) + ) ~> withPermissions( deployRoute(), Permission.Write ) ~> check { diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestingApiHttpServiceSpec.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestingApiHttpServiceSpec.scala index 2c7d0e62824..7ff5b9aa59e 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestingApiHttpServiceSpec.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/api/TestingApiHttpServiceSpec.scala @@ -236,17 +236,21 @@ class TestingApiHttpServiceSpec | "possibleValues": [ | { | "expression": "", - | "label": "" + | "label": "", + | "hintText" : null | }, | { | "expression": "'uno'", - | "label": "uno" + | "label": "uno", + | "hintText" : null | }, | { | "expression": "'due'", - | "label": "due" + | "label": "due", + | "hintText" : null | } | ], + | "mode" : "LIST", | "type": "FixedValuesParameterEditor" | }, | "defaultValue": { diff --git a/docs-internal/api/nu-designer-openapi.yaml b/docs-internal/api/nu-designer-openapi.yaml index 819f3508bd0..f19fc51202d 100644 --- a/docs-internal/api/nu-designer-openapi.yaml +++ b/docs-internal/api/nu-designer-openapi.yaml @@ -221,88 +221,6 @@ paths: security: - {} - httpAuth: [] - /api/processManagement/customAction/{scenarioName}/validation: - post: - tags: - - CustomAction - summary: Endpoint to validate input in custom action fields - operationId: postApiProcessmanagementCustomactionScenarionameValidation - parameters: - - name: Nu-Impersonate-User-Identity - in: header - required: false - schema: - type: - - string - - 'null' - - name: scenarioName - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CustomActionRequest' - required: true - responses: - '200': - description: '' - content: - application/json: - schema: - $ref: '#/components/schemas/CustomActionValidationDto' - '400': - description: 'Invalid value for: header Nu-Impersonate-User-Identity, Invalid - value for: body' - content: - text/plain: - schema: - type: string - '401': - description: '' - content: - text/plain: - schema: - type: string - examples: - CannotAuthenticateUser: - value: The supplied authentication is invalid - ImpersonatedUserNotExistsError: - value: No impersonated user data found for provided identity - '403': - description: '' - content: - text/plain: - schema: - type: string - examples: - InsufficientPermission: - value: The supplied authentication is not authorized to access this - resource - ImpersonationMissingPermission: - value: The supplied authentication is not authorized to impersonate - '404': - description: '' - content: - text/plain: - schema: - type: string - '501': - description: Impersonation is not supported for defined authentication mechanism - content: - text/plain: - schema: - type: string - examples: - Example: - summary: Cannot authenticate impersonated user as impersonation - is not supported by the authentication mechanism - value: Provided authentication method does not support impersonation - security: - - {} - - httpAuth: [] /api/app/healthCheck: get: tags: @@ -1096,6 +1014,88 @@ paths: security: - {} - httpAuth: [] + /api/processManagement/customAction/{scenarioName}/validation: + post: + tags: + - CustomAction + summary: Endpoint to validate input in custom action fields + operationId: postApiProcessmanagementCustomactionScenarionameValidation + parameters: + - name: Nu-Impersonate-User-Identity + in: header + required: false + schema: + type: + - string + - 'null' + - name: scenarioName + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CustomActionRequest' + required: true + responses: + '200': + description: '' + content: + application/json: + schema: + $ref: '#/components/schemas/CustomActionValidationDto' + '400': + description: 'Invalid value for: header Nu-Impersonate-User-Identity, Invalid + value for: body' + content: + text/plain: + schema: + type: string + '401': + description: '' + content: + text/plain: + schema: + type: string + examples: + CannotAuthenticateUser: + value: The supplied authentication is invalid + ImpersonatedUserNotExistsError: + value: No impersonated user data found for provided identity + '403': + description: '' + content: + text/plain: + schema: + type: string + examples: + InsufficientPermission: + value: The supplied authentication is not authorized to access this + resource + ImpersonationMissingPermission: + value: The supplied authentication is not authorized to impersonate + '404': + description: '' + content: + text/plain: + schema: + type: string + '501': + description: Impersonation is not supported for defined authentication mechanism + content: + text/plain: + schema: + type: string + examples: + Example: + summary: Cannot authenticate impersonated user as impersonation + is not supported by the authentication mechanism + value: Provided authentication method does not support impersonation + security: + - {} + - httpAuth: [] /api/migrate: post: tags: @@ -5257,6 +5257,10 @@ components: type: string label: type: string + hintText: + type: + - string + - 'null' FixedExpressionValueWithIcon: title: FixedExpressionValueWithIcon type: object @@ -5274,11 +5278,15 @@ components: FixedValuesParameterEditor: title: FixedValuesParameterEditor type: object + required: + - mode properties: possibleValues: type: array items: $ref: '#/components/schemas/FixedExpressionValue' + mode: + type: string FixedValuesWithIconParameterEditor: title: FixedValuesWithIconParameterEditor type: object @@ -5467,8 +5475,8 @@ components: type: object additionalProperties: $ref: '#/components/schemas/Map_TypingResultInJson' - Map_NodeId_NodeDeploymentData: - title: Map_NodeId_NodeDeploymentData + Map_NodeId_String: + title: Map_NodeId_String type: object additionalProperties: type: string @@ -6348,7 +6356,7 @@ components: scenarioName: type: string nodesDeploymentData: - $ref: '#/components/schemas/Map_NodeId_NodeDeploymentData' + $ref: '#/components/schemas/Map_NodeId_String' comment: type: - string diff --git a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala index 8823c9326e3..efe4e4cb281 100644 --- a/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala +++ b/engine/flink/components-api/src/main/scala/pl/touk/nussknacker/engine/flink/api/process/FlinkCustomNodeContext.scala @@ -2,7 +2,7 @@ package pl.touk.nussknacker.engine.flink.api.process import org.apache.flink.api.common.functions.RuntimeContext import org.apache.flink.api.common.typeinfo.TypeInformation -import pl.touk.nussknacker.engine.api.component.NodeDeploymentData +import pl.touk.nussknacker.engine.api.component.NodesDeploymentData.NodeDeploymentData import pl.touk.nussknacker.engine.api.context.ValidationContext import pl.touk.nussknacker.engine.api.process.ComponentUseCase import pl.touk.nussknacker.engine.api.runtimecontext.EngineRuntimeContext diff --git a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala index 1482658f7bb..5e996846743 100644 --- a/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala +++ b/engine/flink/components-utils/src/main/scala/pl/touk/nussknacker/engine/flink/util/source/CollectionSource.scala @@ -26,10 +26,18 @@ case class CollectionSource[T]( ) extends StandardFlinkSource[T] with ReturningType { - @silent("deprecated") override def sourceStream( env: StreamExecutionEnvironment, flinkNodeContext: FlinkCustomNodeContext + ): DataStreamSource[T] = { + createSourceStream(list, env, flinkNodeContext) + } + + @silent("deprecated") + protected def createSourceStream[T]( + list: List[T], + env: StreamExecutionEnvironment, + flinkNodeContext: FlinkCustomNodeContext ): DataStreamSource[T] = { val typeInformation = TypeInformationDetection.instance.forType[T](returnType) boundedness match { diff --git a/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala b/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala index 36e2b17f08f..8857b738315 100644 --- a/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala +++ b/engine/flink/components/base-tests/src/test/scala/pl/touk/nussknacker/engine/flink/table/source/TableSourceTest.scala @@ -7,11 +7,12 @@ import org.scalatest.LoneElement import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, NodesDeploymentData, SqlFilteringExpression} +import pl.touk.nussknacker.engine.api.component.{ComponentDefinition, NodesDeploymentData} import pl.touk.nussknacker.engine.api.process.ProcessObjectDependencies import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.flink.table.FlinkTableComponentProvider import pl.touk.nussknacker.engine.flink.table.definition.{FlinkDataDefinition, StubbedCatalogFactory} +import pl.touk.nussknacker.engine.flink.table.source.TableSource.SQL_EXPRESSION_PARAMETER_NAME import pl.touk.nussknacker.engine.flink.test.FlinkSpec import pl.touk.nussknacker.engine.flink.util.test.FlinkTestScenarioRunner import pl.touk.nussknacker.engine.process.FlinkJobConfig.ExecutionMode @@ -87,7 +88,7 @@ class TableSourceTest val result = runner .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = true"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = true"))) ) .validValue result.errors shouldBe empty @@ -127,7 +128,7 @@ class TableSourceTest val resultWithoutFiltering = runnerWithCatalogConfiguration .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = true"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = true"))) ) .validValue resultWithoutFiltering.errors shouldBe empty @@ -136,7 +137,7 @@ class TableSourceTest val resultWithFiltering = runnerWithCatalogConfiguration .runWithoutData[Row]( scenario, - nodesData = NodesDeploymentData(Map(NodeId("start") -> SqlFilteringExpression("true = false"))) + nodesData = NodesDeploymentData(Map(NodeId("start") -> Map(SQL_EXPRESSION_PARAMETER_NAME -> "true = false"))) ) .validValue resultWithFiltering.errors shouldBe empty diff --git a/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java b/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java index a6329c746e5..f81ee47c769 100644 --- a/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java +++ b/engine/flink/components/base-unbounded/src/main/java/pl/touk/nussknacker/engine/flink/util/transformer/aggregate/AggregateHelper.java @@ -6,10 +6,11 @@ import pl.touk.nussknacker.engine.api.Hidden; import pl.touk.nussknacker.engine.api.ParamName; import pl.touk.nussknacker.engine.api.definition.DualParameterEditor; -import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue; +import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue$; import pl.touk.nussknacker.engine.api.definition.FixedValuesParameterEditor; import pl.touk.nussknacker.engine.api.definition.SimpleParameterEditor; import pl.touk.nussknacker.engine.api.editor.DualEditorMode; +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode; import scala.collection.JavaConverters; /** @@ -22,17 +23,21 @@ */ public class AggregateHelper implements Serializable { - public static final SimpleParameterEditor SIMPLE_EDITOR = new FixedValuesParameterEditor(JavaConverters.collectionAsScalaIterableConverter(Arrays.asList( - new FixedExpressionValue("#AGG.first", "First"), - new FixedExpressionValue("#AGG.last", "Last"), - new FixedExpressionValue("#AGG.countWhen", "CountWhen"), - new FixedExpressionValue("#AGG.average", "Average"), - new FixedExpressionValue("#AGG.min", "Min"), - new FixedExpressionValue("#AGG.max", "Max"), - new FixedExpressionValue("#AGG.sum", "Sum"), - new FixedExpressionValue("#AGG.list", "List"), - new FixedExpressionValue("#AGG.set", "Set"), - new FixedExpressionValue("#AGG.approxCardinality", "ApproximateSetCardinality"))).asScala().toList()); + public static final SimpleParameterEditor SIMPLE_EDITOR = new FixedValuesParameterEditor( + JavaConverters.collectionAsScalaIterableConverter(Arrays.asList( + FixedExpressionValue$.MODULE$.apply("#AGG.first", "First"), + FixedExpressionValue$.MODULE$.apply("#AGG.last", "Last"), + FixedExpressionValue$.MODULE$.apply("#AGG.countWhen", "CountWhen"), + FixedExpressionValue$.MODULE$.apply("#AGG.average", "Average"), + FixedExpressionValue$.MODULE$.apply("#AGG.min", "Min"), + FixedExpressionValue$.MODULE$.apply("#AGG.max", "Max"), + FixedExpressionValue$.MODULE$.apply("#AGG.sum", "Sum"), + FixedExpressionValue$.MODULE$.apply("#AGG.list", "List"), + FixedExpressionValue$.MODULE$.apply("#AGG.set", "Set"), + FixedExpressionValue$.MODULE$.apply("#AGG.approxCardinality", "ApproximateSetCardinality") + )).asScala().toList(), + FixedValuesEditorMode.LIST + ); @Hidden public static final DualParameterEditor DUAL_EDITOR = new DualParameterEditor(SIMPLE_EDITOR, DualEditorMode.SIMPLE); diff --git a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala index 5b826f57979..5f475f58314 100644 --- a/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala +++ b/engine/flink/components/table/src/main/scala/pl/touk/nussknacker/engine/flink/table/source/TableSource.scala @@ -6,7 +6,6 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment import org.apache.flink.table.api.{DataTypes, Schema} import org.apache.flink.table.catalog.Column.{ComputedColumn, MetadataColumn, PhysicalColumn} import org.apache.flink.types.Row -import pl.touk.nussknacker.engine.api.component.SqlFilteringExpression import pl.touk.nussknacker.engine.api.definition.Parameter import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.api.process.{ @@ -52,7 +51,8 @@ class TableSource( val selectQuery = tableEnv.from(tableDefinition.tableId.toString) val finalQuery = flinkNodeContext.nodeDeploymentData - .map { case SqlFilteringExpression(sqlExpression) => + .flatMap(_.get(SQL_EXPRESSION_PARAMETER_NAME)) + .collect { case sqlExpression => tableEnv.executeSql( s"CREATE TEMPORARY VIEW $filteringInternalViewName AS SELECT * FROM ${tableDefinition.tableId} WHERE $sqlExpression" ) @@ -126,5 +126,6 @@ class TableSource( } object TableSource { + val SQL_EXPRESSION_PARAMETER_NAME = "sqlExpression" private val filteringInternalViewName = "filteringView" } diff --git a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala index ee856a4e369..bd62db8b5a0 100644 --- a/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala +++ b/engine/flink/kafka-components-utils/src/main/scala/pl/touk/nussknacker/engine/kafka/source/flink/FlinkKafkaSource.scala @@ -10,10 +10,17 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaConsumerBase} import org.apache.kafka.clients.consumer.ConsumerRecord import pl.touk.nussknacker.engine.api.NodeId -import pl.touk.nussknacker.engine.api.definition.Parameter +import pl.touk.nussknacker.engine.api.component.ParameterConfig +import pl.touk.nussknacker.engine.api.definition.{FixedExpressionValue, FixedValuesParameterEditor, Parameter} +import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName import pl.touk.nussknacker.engine.api.namespaces.NamingStrategy import pl.touk.nussknacker.engine.api.parameter.ParameterName -import pl.touk.nussknacker.engine.api.process.{ContextInitializer, TestWithParametersSupport, TopicName} +import pl.touk.nussknacker.engine.api.process.{ + ContextInitializer, + TestWithParametersSupport, + TopicName, + WithActivityParameters +} import pl.touk.nussknacker.engine.api.runtimecontext.{ContextIdGenerator, EngineRuntimeContext} import pl.touk.nussknacker.engine.api.test.{TestRecord, TestRecordParser} import pl.touk.nussknacker.engine.flink.api.exception.ExceptionHandler @@ -54,7 +61,8 @@ class FlinkKafkaSource[T]( with Serializable with FlinkSourceTestSupport[T] with RecordFormatterBaseTestDataGenerator - with TestWithParametersSupport[T] { + with TestWithParametersSupport[T] + with WithActivityParameters { @silent("deprecated") override def sourceStream( @@ -72,11 +80,40 @@ class FlinkKafkaSource[T]( protected lazy val topics: NonEmptyList[TopicName.ForSource] = preparedTopics.map(_.prepared) + private val OFFSET_RESET_STRATEGY_PARAM_NAME = "offsetResetStrategy" + + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = { + val defaultValue = if (kafkaConfig.forceLatestRead.contains(true)) Some("LATEST") else Some("NONE") + val editor = Some( + FixedValuesParameterEditor( + List( + FixedExpressionValue("LATEST", "LATEST"), + FixedExpressionValue("EARLIEST", "EARLIEST"), + FixedExpressionValue("NONE", "NONE"), + ) + ) + ) + Map( + ScenarioActionName.Deploy.value -> Map( + OFFSET_RESET_STRATEGY_PARAM_NAME -> ParameterConfig( + defaultValue = defaultValue, + editor = editor, + validators = None, + label = None, + hintText = None + ), + ) + ) + } + @silent("deprecated") protected def flinkSourceFunction( consumerGroupId: String, flinkNodeContext: FlinkCustomNodeContext ): SourceFunction[T] = { + // TODO: use deployment parameters -> offsetResetStrategy + val offsetResetStrategy = + flinkNodeContext.nodeDeploymentData.flatMap(_.get(OFFSET_RESET_STRATEGY_PARAM_NAME)).getOrElse() topics.toList.foreach(KafkaUtils.setToLatestOffsetIfNeeded(kafkaConfig, _, consumerGroupId)) createFlinkSource(consumerGroupId, flinkNodeContext) } diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala index 0ce3696e419..9c7326a3164 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/DevProcessConfigCreator.scala @@ -91,12 +91,14 @@ class DevProcessConfigCreator extends ProcessConfigCreator { )(TypeInformation.of(classOf[SampleProduct])) ) ), - "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), - "boundedSource" -> all(BoundedSource), - "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), - "communicationSource" -> categories(DynamicParametersSource), - "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), - "csv-source-lite" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new LiteCsvSource(_))), + "kafka-transaction" -> all(SourceFactory.noParamUnboundedStreamFactory[String](new NoEndingSource)), + "boundedSource" -> all(BoundedSource), + "boundedSourceWithOffset" -> all(BoundedSourceWithOffset), + "boundedSourceWithOtherParam" -> all(DummyBoundedSourceToDelete), + "oneSource" -> categories(SourceFactory.noParamUnboundedStreamFactory[String](new OneSource)), + "communicationSource" -> categories(DynamicParametersSource), + "csv-source" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new CsvSource)), + "csv-source-lite" -> categories(SourceFactory.noParamUnboundedStreamFactory[CsvRecord](new LiteCsvSource(_))), "genericSourceWithCustomVariables" -> categories(GenericSourceWithCustomVariablesSample), "sql-source" -> categories(SqlSource), "classInstanceSource" -> all(new ReturningClassInstanceSource) diff --git a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala index 05a090f6347..b225fd2be75 100644 --- a/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala +++ b/engine/flink/management/dev-model/src/main/scala/pl/touk/nussknacker/engine/management/sample/source/BoundedSource.scala @@ -1,9 +1,20 @@ package pl.touk.nussknacker.engine.management.sample.source -import pl.touk.nussknacker.engine.api.component.UnboundedStreamComponent -import pl.touk.nussknacker.engine.api.process.SourceFactory +import org.apache.flink.streaming.api.datastream.DataStreamSource +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment +import pl.touk.nussknacker.engine.api.component.{ParameterConfig, UnboundedStreamComponent} +import pl.touk.nussknacker.engine.api.definition.{ + BoolParameterEditor, + FixedExpressionValue, + FixedValuesParameterEditor, + RawParameterEditor +} +import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName +import pl.touk.nussknacker.engine.api.editor.FixedValuesEditorMode +import pl.touk.nussknacker.engine.api.process.{SourceFactory, WithActivityParameters} import pl.touk.nussknacker.engine.api.typed.typing.Unknown import pl.touk.nussknacker.engine.api.{MethodToInvoke, ParamName} +import pl.touk.nussknacker.engine.flink.api.process.FlinkCustomNodeContext import pl.touk.nussknacker.engine.flink.util.source.CollectionSource import scala.jdk.CollectionConverters._ @@ -15,3 +26,83 @@ object BoundedSource extends SourceFactory with UnboundedStreamComponent { new CollectionSource[Any](elements.asScala.toList, None, Unknown) } + +object BoundedSourceWithOffset extends SourceFactory with UnboundedStreamComponent { + + @MethodToInvoke + def source(@ParamName("elements") elements: java.util.List[Any]) = + new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { + + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = { + val fixedValuesEditor = Some( + FixedValuesParameterEditor( + List( + FixedExpressionValue("Continue", "Continue", Some("Resumes reading data where it previously stopped.")), + FixedExpressionValue("Reset", "Reset", Some("Starts reading new events only.")), + FixedExpressionValue("Restart", "Restart", Some("Rewinds reading from the earliest event.")), + ), + FixedValuesEditorMode.RADIO + ) + ) + Map( + ScenarioActionName.Deploy.value -> Map( + "offset" -> ParameterConfig( + defaultValue = None, + editor = Some(RawParameterEditor), + validators = None, + label = Some("Offset"), + hintText = Some( + "Set offset to setup source to emit elements from specified start point in input collection. Empty field resets collection to the beginning." + ) + ), + // TODO: remove offsetResetStrategy + "offsetResetStrategy" -> ParameterConfig( + defaultValue = Some("Restart"), + editor = fixedValuesEditor, + validators = None, + label = Some("Starting point strategy"), + hintText = Some("Example of parameter with fixed values") + ), + ) + ) + } + + override protected def createSourceStream[T]( + list: List[T], + env: StreamExecutionEnvironment, + flinkNodeContext: FlinkCustomNodeContext + ): DataStreamSource[T] = { + val offsetOpt = flinkNodeContext.nodeDeploymentData.flatMap(_.get("offset")) + val elementsWithOffset = offsetOpt match { + case Some(offset) => list.drop(offset.toInt) + case _ => list + } + super.createSourceStream(elementsWithOffset, env, flinkNodeContext) + } + + } + +} + +object DummyBoundedSourceToDelete extends SourceFactory with UnboundedStreamComponent { + + @MethodToInvoke + def source(@ParamName("elements") elements: java.util.List[Any]) = + new CollectionSource[Any](elements.asScala.toList, None, Unknown) with WithActivityParameters { + + override def activityParametersDefinition: Map[String, Map[String, ParameterConfig]] = + Map( + ScenarioActionName.Deploy.value -> Map( + "otherParameter" -> ParameterConfig( + defaultValue = None, + editor = Some(BoolParameterEditor), + validators = None, + label = Some("Other parameter"), + hintText = Some("this is hint") + ) + ) + ) + + } + +} diff --git a/nussknacker-dist/src/universal/conf/dev-application.conf b/nussknacker-dist/src/universal/conf/dev-application.conf index ddf3c4c27aa..3082ff0a08c 100644 --- a/nussknacker-dist/src/universal/conf/dev-application.conf +++ b/nussknacker-dist/src/universal/conf/dev-application.conf @@ -359,8 +359,8 @@ commentSettings: { } deploymentCommentSettings: { - validationPattern: "(.*)" - exampleComment: "issues/1234" + validationPattern: "issues/\\d+" + exampleComment: "issues/1234 Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec ultricies nec sapien id bibendum. Ut in mollis risus. Curabitur efficitur maximus interdum. Vivamus convallis eu nibh ut rhoncus. Quisque finibus maximus dui vel finibus." } countsSettings { diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala new file mode 100644 index 00000000000..090d9dbf425 --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ActivityInfoProvider.scala @@ -0,0 +1,11 @@ +package pl.touk.nussknacker.engine.definition.activity + +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.component.ParameterConfig +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess + +trait ActivityInfoProvider { + + def getActivityParameters(processVersion: ProcessVersion, scenario: CanonicalProcess): Map[String, Map[String, Map[String, ParameterConfig]]] + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala new file mode 100644 index 00000000000..97a6e50901b --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/CommonModelDataInfoProvider.scala @@ -0,0 +1,40 @@ +package pl.touk.nussknacker.engine.definition.activity + +import cats.data.ValidatedNel +import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.context.ProcessCompilationError +import pl.touk.nussknacker.engine.api.process.{ComponentUseCase, Source} +import pl.touk.nussknacker.engine.api.{JobData, NodeId, process} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.compile.ExpressionCompiler +import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} +import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor +import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} +import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector + +class CommonModelDataInfoProvider(modelData: ModelData) { + + private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper + + private lazy val nodeCompiler = new NodeCompiler( + modelData.modelDefinition, + new FragmentParametersDefinitionExtractor(modelData.modelClassLoader.classLoader), + expressionCompiler, + modelData.modelClassLoader.classLoader, + Seq.empty, + ProductionServiceInvocationCollector, + ComponentUseCase.TestDataGeneration, + nonServicesLazyParamStrategy = LazyParameterCreationStrategy.default + ) + + protected def prepareSourceObj( + source: SourceNodeData + )(implicit jobData: JobData, nodeId: NodeId): ValidatedNel[ProcessCompilationError, Source] = { + nodeCompiler.compileSource(source).compiledObject + } + + protected def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { + scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) + } + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala new file mode 100644 index 00000000000..55d9d556333 --- /dev/null +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/activity/ModelDataActivityInfoProvider.scala @@ -0,0 +1,54 @@ +package pl.touk.nussknacker.engine.definition.activity + +import cats.data.Validated.Valid +import pl.touk.nussknacker.engine.ModelData +import pl.touk.nussknacker.engine.api.component.ParameterConfig +import pl.touk.nussknacker.engine.api.process.WithActivityParameters +import pl.touk.nussknacker.engine.api.{JobData, MetaData, NodeId, ProcessVersion} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.graph.node.SourceNodeData +import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap + +class ModelDataActivityInfoProvider(modelData: ModelData) + extends CommonModelDataInfoProvider(modelData) + with ActivityInfoProvider { + + override def getActivityParameters( + processVersion: ProcessVersion, + scenario: CanonicalProcess + ): Map[String, Map[String, Map[String, ParameterConfig]]] = { + val jobData = JobData(scenario.metaData, processVersion) + modelData.withThisAsContextClassLoader { + val nodeToActivityToParameters = collectAllSources(scenario) + .map(source => source.id -> getActivityParameters(source, jobData)) + .toMap + groupByActivity(nodeToActivityToParameters) + } + } + + private def groupByActivity( + nodeToActivityToParameters: Map[String, Map[String, Map[String, ParameterConfig]]] + ): Map[String, Map[String, Map[String, ParameterConfig]]] = { + val activityToNodeToParameters = for { + (node, activityToParams) <- nodeToActivityToParameters.toList + (activity, params) <- activityToParams.toList + } yield (activity, node -> params) + activityToNodeToParameters + .groupBy(_._1) + .mapValuesNow(_.map(_._2).toMap) + } + + private def getActivityParameters( + source: SourceNodeData, + jobData: JobData + ): Map[String, Map[String, ParameterConfig]] = { + modelData.withThisAsContextClassLoader { + val compiledSource = prepareSourceObj(source)(jobData, NodeId(source.id)) + compiledSource match { + case Valid(s: WithActivityParameters) => s.activityParametersDefinition + case _ => Map.empty + } + } + } + +} diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala index ae97173b9a6..c5b81c6f965 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorBasedLanguageDeterminer.scala @@ -16,7 +16,7 @@ object EditorBasedLanguageDeterminer { editor match { case BoolParameterEditor | StringParameterEditor | DateParameterEditor | TimeParameterEditor | DateTimeParameterEditor | TextareaParameterEditor | JsonParameterEditor | DurationParameterEditor(_) | - PeriodParameterEditor(_) | CronParameterEditor | FixedValuesParameterEditor(_) | + PeriodParameterEditor(_) | CronParameterEditor | FixedValuesParameterEditor(_, _) | FixedValuesWithIconParameterEditor(_) => Expression.Language.Spel case SqlParameterEditor | SpelTemplateParameterEditor => diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala index b1323d78997..78a06461f21 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/defaults/EditorPossibleValuesBasedDefaultValueDeterminer.scala @@ -15,10 +15,10 @@ protected object EditorPossibleValuesBasedDefaultValueDeterminer extends Paramet override def determineParameterDefaultValue(parameters: DefaultValueDeterminerParameters): Option[Expression] = { parameters.determinedEditor .flatMap { - case FixedValuesParameterEditor(firstValue :: _) => Some(Expression.spel(firstValue.expression)) + case FixedValuesParameterEditor(firstValue :: _, _) => Some(Expression.spel(firstValue.expression)) // it is better to see error that field is not filled instead of strange default value like '' for String - case FixedValuesParameterEditor(Nil) => Some(Expression.spel("")) - case DualParameterEditor(FixedValuesParameterEditor(firstValue :: _), _) => + case FixedValuesParameterEditor(Nil, _) => Some(Expression.spel("")) + case DualParameterEditor(FixedValuesParameterEditor(firstValue :: _, _), _) => Some(Expression.spel(firstValue.expression)) case TabularTypedDataEditor => Some(Expression.tabularDataDefinition(TabularTypedData.empty.stringify)) diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala index 8a7230026b5..b1096f613b1 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/component/parameter/validator/EditorBasedValidatorExtractor.scala @@ -5,7 +5,7 @@ import pl.touk.nussknacker.engine.api.definition._ object EditorBasedValidatorExtractor extends ValidatorExtractor { override def extract(params: ValidatorExtractorParameters): Option[ParameterValidator] = { - params.extractedEditor.collect { case FixedValuesParameterEditor(possibleValues) => + params.extractedEditor.collect { case FixedValuesParameterEditor(possibleValues, _) => FixedValuesValidator(possibleValues) } } diff --git a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala index d2c0b815744..b8e99f5a18b 100644 --- a/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala +++ b/scenario-compiler/src/main/scala/pl/touk/nussknacker/engine/definition/test/ModelDataTestInfoProvider.scala @@ -12,13 +12,17 @@ import pl.touk.nussknacker.engine.api.{JobData, NodeId, ProcessVersion} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.compile.ExpressionCompiler import pl.touk.nussknacker.engine.compile.nodecompilation.{LazyParameterCreationStrategy, NodeCompiler} +import pl.touk.nussknacker.engine.definition.activity.CommonModelDataInfoProvider import pl.touk.nussknacker.engine.definition.fragment.FragmentParametersDefinitionExtractor -import pl.touk.nussknacker.engine.graph.node.{SourceNodeData, asFragmentInputDefinition, asSource} +import pl.touk.nussknacker.engine.graph.node.SourceNodeData import pl.touk.nussknacker.engine.resultcollector.ProductionServiceInvocationCollector import pl.touk.nussknacker.engine.util.ListUtil import shapeless.syntax.typeable._ -class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider with LazyLogging { +class ModelDataTestInfoProvider(modelData: ModelData) + extends CommonModelDataInfoProvider(modelData) + with TestInfoProvider + with LazyLogging { private lazy val expressionCompiler = ExpressionCompiler.withoutOptimization(modelData).withLabelsDictTyper @@ -137,12 +141,6 @@ class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider w .getOrElse(Left("Scenario doesn't have any valid source supporting test data generation")) } - private def prepareSourceObj( - source: SourceNodeData - )(implicit jobData: JobData, nodeId: NodeId): ValidatedNel[ProcessCompilationError, Source] = { - nodeCompiler.compileSource(source).compiledObject - } - private def generateTestData(generators: NonEmptyList[(NodeId, TestDataGenerator)], size: Int) = { modelData.withThisAsContextClassLoader { val sourceTestDataList = generators.map { case (sourceId, testDataGenerator) => @@ -177,10 +175,6 @@ class ModelDataTestInfoProvider(modelData: ModelData) extends TestInfoProvider w .map(scenarioTestRecords => ScenarioTestData(scenarioTestRecords.toList)) } - private def collectAllSources(scenario: CanonicalProcess): List[SourceNodeData] = { - scenario.collectAllNodes.flatMap(asSource) ++ scenario.collectAllNodes.flatMap(asFragmentInputDefinition) - } - private def formatError(error: String, recordIdx: Int): String = { s"Record ${recordIdx + 1} - $error" }