From 0245900f82649bdf3d2521cc71748620a710e5c2 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 19 Nov 2024 20:25:04 -0800 Subject: [PATCH 01/17] init --- .../metadata/OperatorGroupConstants.scala | 4 +- .../workflow/common/operators/LogicalOp.scala | 70 +++--------------- .../ifStatement/IfStatementOpDesc.scala | 61 +++++++++++++++ .../ifStatement/IfStatementOpExec.scala | 19 +++++ .../assets/operator_images/IfStatement.png | Bin 0 -> 7020 bytes 5 files changed, 92 insertions(+), 62 deletions(-) create mode 100644 core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala create mode 100644 core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala create mode 100644 core/gui/src/assets/operator_images/IfStatement.png diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala index 095c37cfbeb..27aafccaa69 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala @@ -21,6 +21,7 @@ object OperatorGroupConstants { final val JAVA_GROUP = "Java" final val R_GROUP = "R" final val MACHINE_LEARNING_GENERAL_GROUP = "Machine Learning General" + final val CONTROL_GROUP = "Control Block" /** * The order of the groups to show up in the frontend operator panel. @@ -46,6 +47,7 @@ object OperatorGroupConstants { GroupInfo(UTILITY_GROUP), GroupInfo(API_GROUP), GroupInfo(UDF_GROUP, List(GroupInfo(PYTHON_GROUP), GroupInfo(JAVA_GROUP), GroupInfo(R_GROUP))), - GroupInfo(VISUALIZATION_GROUP) + GroupInfo(VISUALIZATION_GROUP), + GroupInfo(CONTROL_GROUP) ) } diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala index be68212a96f..ab0bf94e551 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala @@ -1,27 +1,18 @@ package edu.uci.ics.texera.workflow.common.operators import com.fasterxml.jackson.annotation.JsonSubTypes.Type -import com.fasterxml.jackson.annotation.{ - JsonIgnore, - JsonProperty, - JsonPropertyDescription, - JsonSubTypes, - JsonTypeInfo -} +import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyDescription, JsonSubTypes, JsonTypeInfo} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import edu.uci.ics.amber.engine.common.executor.OperatorExecutor import edu.uci.ics.amber.engine.common.model.{PhysicalOp, PhysicalPlan, WorkflowContext} import edu.uci.ics.amber.engine.common.model.tuple.Schema -import edu.uci.ics.amber.engine.common.virtualidentity.{ - ExecutionIdentity, - OperatorIdentity, - WorkflowIdentity -} +import edu.uci.ics.amber.engine.common.virtualidentity.{ExecutionIdentity, OperatorIdentity, WorkflowIdentity} import edu.uci.ics.amber.engine.common.workflow.PortIdentity import edu.uci.ics.texera.web.OPversion import edu.uci.ics.texera.workflow.common.metadata.{OperatorInfo, PropertyNameConstants} import edu.uci.ics.texera.workflow.operators.aggregate.AggregateOpDesc import edu.uci.ics.texera.workflow.operators.cartesianProduct.CartesianProductOpDesc +import edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement.IfStatementOpDesc import edu.uci.ics.texera.workflow.operators.dictionary.DictionaryMatcherOpDesc import edu.uci.ics.texera.workflow.operators.difference.DifferenceOpDesc import edu.uci.ics.texera.workflow.operators.distinct.DistinctOpDesc @@ -33,16 +24,8 @@ import edu.uci.ics.texera.workflow.operators.intersect.IntersectOpDesc import edu.uci.ics.texera.workflow.operators.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.texera.workflow.operators.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.texera.workflow.operators.limit.LimitOpDesc -import edu.uci.ics.texera.workflow.operators.huggingFace.{ - HuggingFaceIrisLogisticRegressionOpDesc, - HuggingFaceSentimentAnalysisOpDesc, - HuggingFaceSpamSMSDetectionOpDesc, - HuggingFaceTextSummarizationOpDesc -} -import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.KNNTrainer.{ - SklearnAdvancedKNNClassifierTrainerOpDesc, - SklearnAdvancedKNNRegressorTrainerOpDesc -} +import edu.uci.ics.texera.workflow.operators.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} +import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVRTrainer.SVCTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.texera.workflow.operators.projection.ProjectionOpDesc @@ -51,42 +34,11 @@ import edu.uci.ics.texera.workflow.operators.regex.RegexOpDesc import edu.uci.ics.texera.workflow.operators.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.texera.workflow.operators.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.texera.workflow.operators.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.texera.workflow.operators.sklearn.{ - SklearnAdaptiveBoostingOpDesc, - SklearnBaggingOpDesc, - SklearnBernoulliNaiveBayesOpDesc, - SklearnComplementNaiveBayesOpDesc, - SklearnDecisionTreeOpDesc, - SklearnDummyClassifierOpDesc, - SklearnExtraTreeOpDesc, - SklearnExtraTreesOpDesc, - SklearnGaussianNaiveBayesOpDesc, - SklearnGradientBoostingOpDesc, - SklearnKNNOpDesc, - SklearnLinearRegressionOpDesc, - SklearnLinearSVMOpDesc, - SklearnLogisticRegressionCVOpDesc, - SklearnLogisticRegressionOpDesc, - SklearnMultiLayerPerceptronOpDesc, - SklearnMultinomialNaiveBayesOpDesc, - SklearnNearestCentroidOpDesc, - SklearnPassiveAggressiveOpDesc, - SklearnPerceptronOpDesc, - SklearnPredictionOpDesc, - SklearnProbabilityCalibrationOpDesc, - SklearnRandomForestOpDesc, - SklearnRidgeCVOpDesc, - SklearnRidgeOpDesc, - SklearnSDGOpDesc, - SklearnSVMOpDesc -} +import edu.uci.ics.texera.workflow.operators.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} import edu.uci.ics.texera.workflow.operators.sort.SortOpDesc import edu.uci.ics.texera.workflow.operators.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.texera.workflow.operators.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.apis.twitter.v2.{ - TwitterFullArchiveSearchSourceOpDesc, - TwitterSearchSourceOpDesc -} +import edu.uci.ics.texera.workflow.operators.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} import edu.uci.ics.texera.workflow.operators.source.fetcher.URLFetcherOpDesc import edu.uci.ics.texera.workflow.operators.source.scan.FileScanSourceOpDesc import edu.uci.ics.texera.workflow.operators.source.scan.csv.CSVScanSourceOpDesc @@ -101,12 +53,7 @@ import edu.uci.ics.texera.workflow.operators.symmetricDifference.SymmetricDiffer import edu.uci.ics.texera.workflow.operators.typecasting.TypeCastingOpDesc import edu.uci.ics.texera.workflow.operators.udf.java.JavaUDFOpDesc import edu.uci.ics.texera.workflow.operators.udf.python.source.PythonUDFSourceOpDescV2 -import edu.uci.ics.texera.workflow.operators.udf.python.{ - DualInputPortsPythonUDFOpDescV2, - PythonLambdaFunctionOpDesc, - PythonTableReducerOpDesc, - PythonUDFOpDescV2 -} +import edu.uci.ics.texera.workflow.operators.udf.python.{DualInputPortsPythonUDFOpDescV2, PythonLambdaFunctionOpDesc, PythonTableReducerOpDesc, PythonUDFOpDescV2} import edu.uci.ics.texera.workflow.operators.udf.r.{RUDFOpDesc, RUDFSourceOpDesc} import edu.uci.ics.texera.workflow.operators.union.UnionOpDesc import edu.uci.ics.texera.workflow.operators.unneststring.UnnestStringOpDesc @@ -159,6 +106,7 @@ trait StateTransferFunc ) @JsonSubTypes( Array( + new Type(value = classOf[IfStatementOpDesc], name = "IfStatement"), new Type(value = classOf[SankeyDiagramOpDesc], name = "SankeyDiagram"), new Type(value = classOf[IcicleChartOpDesc], name = "IcicleChart"), new Type(value = classOf[CSVScanSourceOpDesc], name = "CSVFileScan"), diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala new file mode 100644 index 00000000000..057b09c4fc4 --- /dev/null +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala @@ -0,0 +1,61 @@ +package edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement + +import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import edu.uci.ics.amber.engine.architecture.deploysemantics.layer.OpExecInitInfo +import edu.uci.ics.amber.engine.common.model.{PhysicalOp, SchemaPropagationFunc} +import edu.uci.ics.amber.engine.common.model.tuple.Schema +import edu.uci.ics.amber.engine.common.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import edu.uci.ics.amber.engine.common.workflow.{InputPort, OutputPort, PortIdentity} +import edu.uci.ics.texera.workflow.common.metadata.{OperatorGroupConstants, OperatorInfo} +import edu.uci.ics.texera.workflow.common.operators.LogicalOp + +class IfStatementOpDesc extends LogicalOp { + @JsonProperty(required = true) + @JsonSchemaTitle("Condition State") + @JsonPropertyDescription("name of the state variable to evaluate") + var stateName: String = _ + + override def getPhysicalOp( + workflowId: WorkflowIdentity, + executionId: ExecutionIdentity + ): PhysicalOp = { + PhysicalOp + .oneToOnePhysicalOp( + workflowId, + executionId, + operatorIdentifier, + OpExecInitInfo((_, _) => { + new IfStatementOpExec(stateName) + }) + ) + .withInputPorts(operatorInfo.inputPorts) + .withOutputPorts(operatorInfo.outputPorts) + .withParallelizable(false) + .withPropagateSchema( + SchemaPropagationFunc(inputSchemas => + operatorInfo.outputPorts + .map(_.id) + .map(id => id -> inputSchemas(operatorInfo.inputPorts.last.id)) + .toMap + ) + ) + } + + override def operatorInfo: OperatorInfo = + OperatorInfo( + "If Statement", + "If Statement", + OperatorGroupConstants.CONTROL_GROUP, + inputPorts = List( + InputPort(PortIdentity(), "Condition(State)"), + InputPort(PortIdentity(1), "Data", dependencies = List(PortIdentity()))), + outputPorts = List( + OutputPort(PortIdentity(), "True"), + OutputPort(PortIdentity(1), "False")) + ) + + override def getOutputSchema(schemas: Array[Schema]): Schema = throw new NotImplementedError() + + override def getOutputSchemas(schemas: Array[Schema]): Array[Schema] = Array(schemas(1), schemas(1)) +} diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala new file mode 100644 index 00000000000..14c64650d79 --- /dev/null +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala @@ -0,0 +1,19 @@ +package edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement + +import edu.uci.ics.amber.engine.common.executor.OperatorExecutor +import edu.uci.ics.amber.engine.common.model.State +import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} +import edu.uci.ics.amber.engine.common.workflow.PortIdentity + +class IfStatementOpExec(stateName: String) extends OperatorExecutor { + private var output_port: Int = _ + + override def processState(state: State, port: Int): Option[State] = { + output_port = if (state.get(stateName).asInstanceOf[Boolean]) 0 else 1 + None + } + + override def processTupleMultiPort(tuple: Tuple, port: Int): Iterator[(TupleLike, Option[PortIdentity])] = Iterator((tuple, Some(PortIdentity(output_port)))) + + override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = ??? +} diff --git a/core/gui/src/assets/operator_images/IfStatement.png b/core/gui/src/assets/operator_images/IfStatement.png new file mode 100644 index 0000000000000000000000000000000000000000..5808af116b04012394690205c0253b4a42a23546 GIT binary patch literal 7020 zcmeHM>0eXV*4~M!1X~&ERcnDzB!E^#s#QQhY*8^xaRvtlE5fC;GRdqa7TZvr}z}-=(^;+1M4sGjCd=TOFEL?-+XwwLFlx3CeIhX>To=D=-m0f zhwIaQ4bE^3zI=bT_~6|Yi(YI!dPorztm;e9hFyF+QfX77TWOheAR$KdpeC=TJS-{6 zk*o}8+TcvcLnjs?CLa>=huQx={GWrcyf9Jr>5gs_{quQ5)~j+KzR5Qg4y$9{nGM#B zn?D#m>#=YyA#nsq-07Z4$i^!R6fM&BZ*>B8sV%i` zs(nqDud_(t;aH1=HLH5^5guvA82~`b#*sN?UERwbM)w{;_fhd zJ*899JS(Jku=ba=AWy3F{@zGyo4ZC#$jKX9@8xL* zeVzFW*}QGeU$t-u$yw$!ChQ(E)%}rPcSw<(dopz|%)dSD7H#v)wsJr*?;PpdW4@D05VdNZZdP{;t=)NU^KPtvD4(w8mN` zd_irWJ^1j)x0VSzMpI(I!{U1{_=`NFiu%nQ%hYlgL##A?s7H5t4XgVNb3P%&_xmRT z|6|I`R@d0BILhf%;>~khLSny;qh?!Gt{y#V@~nSIh&vt256S6kQjK6V#cKv3N3J+3 zJgD~S#NXFG4wcX|Qsa&pVD%DKrR~L7^)uD}NJ!W33%iHti+o`rZ-ao42tIU7pmH|p z&k1!@gj*&=y-bVJ?pjHVOHMx>3itr8H%>cERa=j&&<~W44U7=6t(@7Qy+eVMh$GeRihm@kz{buIhT2+Qn z1wQ)OpOzA0U*nujchg3ZYE#OH5btm8HQ~O_L1$F_H*1F*`^`355%SUBg5kq_yYsxo zl$_=Q3u?EqHcmKP+;0|bh6pEBW(Z0Gs&^Hu_;DV?e`6IK8^3C$D57?h;^BJq&m6L{ zE;B;ixm5A8)wOpmqI9T`^0BJiGkWyrtI~E0MRcL0F$2tO&f}K~qH>|8gT@Cg@@luP zgpL&C`8Wr?fDo70`~*KL3@?A~D0pEkq21N%Wn_PZI{M>1gk**bsRve-TOhPn8mpo; z{bn9dr;@-&IOW=#qzG_y*tYp%1ir^wObeBrqF`DwjSveVxB( zUdJV0>`MU3VgDHXC~Scum#QvH{GCd8kCa)5q|5W=U6AY0ls9=KP2j)D`HT*Z%xU@= zqHFN#?SdCamG!NziGZj#9IsGGtIx!{`EZD=I@es4OSc}5J8~sM87uJL=6uF(1&3=J zkkm62!iUL6D(qBkq9zIbt+cv0acwEiXg&~XeHM0nc7F@?n4gd^dNfa$h!Y;=D1Q=^ zY;X?ZOXyWxA|CU-(JdO_m!9W+$AqgILB|Q@?N-+{ZSGM;{L*6HmpxU4oNMlGzFo;8 zSy%BprF5VCJ(XWt!n+a5te0T@moc8UGfghKjJrRdE}>_p#vK_|@vj}hQZ<`o<<$th z*ACYeN{)Q)G*)*mksH!DoyaQw_8Pw0_-2USVyrC(m(KN^02fL(!(Nk6&tahcxG8%Q zw^;)J?aqhfJyXfaj>^S@R`k0_b#oda=VMALtmp*{DMW2o>g1weTzP(p(-{3VZ2Q`( zGNDbBnUx(mg{z$w^ z|8@U&?~RzP6t_AoS2!-|l+!tp>NrHs;#(1F`b)-#fzsg6;2|Hxnz+5x45?-xR^gM^ zW)B7?Bgg#Rm>{iQFygGKGn9yq^EGekyPH{T(lvC# zMKwfcGq3h|E6bPX^Zi)}OCtq-Z=1)4>JGHkP2nSsh6Dq<=M%f4YAQrJY8%|%^icn1 zck`Ek&i*>U*!ME+(rb&#Jye{J{?1?UhJAJK2WcgA%+sMh1H2&A?-P|g>i8l!dW5>J zAd%a`nCuOl-(@p!+b<}|Rqn5M%wj?5yPrj>TMY3*HRSRc;$vxsQdlIhHD!}$PllAV zl=181%3lQ9;zaJq)0yOt@?G~<%y)Spr^6!EOOPrPml|tb)Gj%NTT~U7LdRne~{B|lLVu8;}r6B+2^wrP-NrNp^zyYQpTSxcq{8fm=A-c zrovIJ>!Gw95C1q%LkmX@#u27`ze^1U!b@?6ahmA3KP$p$2UD$&aM-8S^%)Xn?>8{q zDHtx7QhqJa{+h^r_i`G^DoU;I);|r)^*;NG@eaKA7iD;%NHp;1WIN5KLmtd+5I3my2MF#VHRF7Jr?|_=#KWbnP7nN55w(5^!b;mr=w<( zA9Bv3GyA&`&w>z{J@4wDoAgL`N_3vbmLT?LQ651ZLTx@mcD4b8?3k-k&>H~I_&z4E ztMMx z!dY%xwB%_Pt#{m20STr>64^&E`?TY?VInC)c`xA3iM@cMm8M#HlQSn4xjSd?bNGl( z5s-_b0))fOt*(Iu2ye>(_6%tJ@Q_CVP>TmqyuG9T;?t#x+~UJVB=CuibPQHIEYJVm zkjR$%S;D1CByT(tvr z!47@8`HYFyw3x>ps5s_B9=A>rul@*PE@Gh0aM4YLGP-zdgDn=$K(hI#K$1^jQ);7|8j$?>3HR9X3p3kylT@3N4+o zimW1Jf8-3J1NPg^_DU6-mhjja9P)>`;mSAoQP;(J`*qVlU=6V{;Da1Ugg;N9M) z-l<|CL$2>a?QlswT;Okka>I}b~gn^>6s-T4YETGBgUSZMBL<%b*j^$rXOhjsXFqf%~9pe6cW4!9njF9 z*LB4|eF$#uvtG#77NIJ0IXa8w;^)lKlc0i7?t%e&jagP}IJQ$j{g3TfWk^a{|4J`d zEpub;*H&=VNKO%J5M4H#3X@E1`?&U>%@zL$p~t4%u}p24Y%*i0yw$bq752pBudUL} z3getuq_SMev7HZL$-qUFCpt>Tmg!#n2Mu_7$t5}WAUSuA=R(%cB_T)IhrXEs)Y zJW|9n=CXaKK;V;*dvm*znXIvgec(zW_s(1-uYH!P%GVea7=3E; z4>XBxFH$FsRGQ9x_6u681}{=>i5l%K<8j|I9+x+Z^2B@)z-YAQEP+>+jORs`Z*sO#kqk<_clpCcvhOq^|i+3D_l96-Auj7Rd=?! z`Z{CDzu-)bFh818y-X2PC@I^9e&?OH;BC;m6Xsn@4OyvZ`4p*VelueS7Fjn?gg6(SCO&Rexqj3j?FgEOH?57dW!G*)Ryh

*awZh7tD>Vk1Wlnn2^M%*swqiWn6FX(?X>SFK9DJlh<&`(W z_#sCe$Mmr&;_W{A&ws*$>qdqh>CwZ>>5Gep;p)$A4Z_NfWV3zGRf2Vh?XOe*i;6a2XIyvv(# ze`RepAa62^*tbfy4X@j6w1@3Iuj8}K9Pv^>a_5?h%ChSsOtjo%8>jse`fO}k@wYHa zj_8n<&h?$cB7dZtu{)SX*N(M-kMY7~7sXFhH4{>*pr|*d3IORA*!)4+aWw z_h&5ztZ)RCdlTcYKFlV7&-fJ=1zJ8_w6=yk>&^S?=;FoC|u>9Fos{PZEG_lp09xgM$ zTDnTMfN4V7IT{RBP~V7iAG${k@E4WxE>97z9e&(uzRU}=yp7uhd!)+U%+&1Vb89wU zTU3^zyvj^Bf^>NK%D6(eZw_N9w;A$nAiu$4?8$!zZfDKd{xGc$FGp|gN{IRzqVkpX z^^UPmF?`+b{8cWb4M3V6DB)zOcvm<%-}C!{8^0a{IeX!#I0mmQZrMUPS|;oqJ!;6z zK3mQ2uZFZen8X}YzE|&(fVwoXp(yJ%*>6usYAR7RgO zVd;AeADz_f;hFiJ94BjqYtf|cEw$2FtmYzK)Tg4XG(DQ{P?$)F-?%pnbY}nYm1kYn zzAr8T_=PZ$z>nkfxvIa;5jE<%P{)Pryp2O4aYE8Mf92#sm{m$RmlGRt<~61e4)#CqF$DOcO}o^4q`x7 zGA4tb!xflz*UgsD*@)eZ^|2JjjQwT@n%_yC5|>2x!X0d_pn7hyn+!|A< Date: Tue, 19 Nov 2024 20:25:55 -0800 Subject: [PATCH 02/17] update --- .../controlBlock/ifStatement/IfStatementOpDesc.scala | 4 ++-- .../controlBlock/ifStatement/IfStatementOpExec.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala index 057b09c4fc4..ecc40c51f04 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala @@ -14,7 +14,7 @@ class IfStatementOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Condition State") @JsonPropertyDescription("name of the state variable to evaluate") - var stateName: String = _ + var conditionName: String = _ override def getPhysicalOp( workflowId: WorkflowIdentity, @@ -26,7 +26,7 @@ class IfStatementOpDesc extends LogicalOp { executionId, operatorIdentifier, OpExecInitInfo((_, _) => { - new IfStatementOpExec(stateName) + new IfStatementOpExec(conditionName) }) ) .withInputPorts(operatorInfo.inputPorts) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala index 14c64650d79..a9563d8d764 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala @@ -5,11 +5,11 @@ import edu.uci.ics.amber.engine.common.model.State import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} import edu.uci.ics.amber.engine.common.workflow.PortIdentity -class IfStatementOpExec(stateName: String) extends OperatorExecutor { +class IfStatementOpExec(conditionName: String) extends OperatorExecutor { private var output_port: Int = _ override def processState(state: State, port: Int): Option[State] = { - output_port = if (state.get(stateName).asInstanceOf[Boolean]) 0 else 1 + output_port = if (state.get(conditionName).asInstanceOf[Boolean]) 0 else 1 None } From 247e43e69050d79c47105ea994cbb341c0a0eca1 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 19 Nov 2024 20:51:00 -0800 Subject: [PATCH 03/17] fix fmt --- .../workflow/common/operators/LogicalOp.scala | 68 +++++++++++++++++-- .../ifStatement/IfStatementOpDesc.scala | 10 +-- .../ifStatement/IfStatementOpExec.scala | 6 +- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala index ab0bf94e551..9218ed446ed 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala @@ -1,12 +1,22 @@ package edu.uci.ics.texera.workflow.common.operators import com.fasterxml.jackson.annotation.JsonSubTypes.Type -import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty, JsonPropertyDescription, JsonSubTypes, JsonTypeInfo} +import com.fasterxml.jackson.annotation.{ + JsonIgnore, + JsonProperty, + JsonPropertyDescription, + JsonSubTypes, + JsonTypeInfo +} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import edu.uci.ics.amber.engine.common.executor.OperatorExecutor import edu.uci.ics.amber.engine.common.model.{PhysicalOp, PhysicalPlan, WorkflowContext} import edu.uci.ics.amber.engine.common.model.tuple.Schema -import edu.uci.ics.amber.engine.common.virtualidentity.{ExecutionIdentity, OperatorIdentity, WorkflowIdentity} +import edu.uci.ics.amber.engine.common.virtualidentity.{ + ExecutionIdentity, + OperatorIdentity, + WorkflowIdentity +} import edu.uci.ics.amber.engine.common.workflow.PortIdentity import edu.uci.ics.texera.web.OPversion import edu.uci.ics.texera.workflow.common.metadata.{OperatorInfo, PropertyNameConstants} @@ -24,8 +34,16 @@ import edu.uci.ics.texera.workflow.operators.intersect.IntersectOpDesc import edu.uci.ics.texera.workflow.operators.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.texera.workflow.operators.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.texera.workflow.operators.limit.LimitOpDesc -import edu.uci.ics.texera.workflow.operators.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} -import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} +import edu.uci.ics.texera.workflow.operators.huggingFace.{ + HuggingFaceIrisLogisticRegressionOpDesc, + HuggingFaceSentimentAnalysisOpDesc, + HuggingFaceSpamSMSDetectionOpDesc, + HuggingFaceTextSummarizationOpDesc +} +import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.KNNTrainer.{ + SklearnAdvancedKNNClassifierTrainerOpDesc, + SklearnAdvancedKNNRegressorTrainerOpDesc +} import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVRTrainer.SVCTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.texera.workflow.operators.projection.ProjectionOpDesc @@ -34,11 +52,42 @@ import edu.uci.ics.texera.workflow.operators.regex.RegexOpDesc import edu.uci.ics.texera.workflow.operators.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.texera.workflow.operators.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.texera.workflow.operators.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.texera.workflow.operators.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} +import edu.uci.ics.texera.workflow.operators.sklearn.{ + SklearnAdaptiveBoostingOpDesc, + SklearnBaggingOpDesc, + SklearnBernoulliNaiveBayesOpDesc, + SklearnComplementNaiveBayesOpDesc, + SklearnDecisionTreeOpDesc, + SklearnDummyClassifierOpDesc, + SklearnExtraTreeOpDesc, + SklearnExtraTreesOpDesc, + SklearnGaussianNaiveBayesOpDesc, + SklearnGradientBoostingOpDesc, + SklearnKNNOpDesc, + SklearnLinearRegressionOpDesc, + SklearnLinearSVMOpDesc, + SklearnLogisticRegressionCVOpDesc, + SklearnLogisticRegressionOpDesc, + SklearnMultiLayerPerceptronOpDesc, + SklearnMultinomialNaiveBayesOpDesc, + SklearnNearestCentroidOpDesc, + SklearnPassiveAggressiveOpDesc, + SklearnPerceptronOpDesc, + SklearnPredictionOpDesc, + SklearnProbabilityCalibrationOpDesc, + SklearnRandomForestOpDesc, + SklearnRidgeCVOpDesc, + SklearnRidgeOpDesc, + SklearnSDGOpDesc, + SklearnSVMOpDesc +} import edu.uci.ics.texera.workflow.operators.sort.SortOpDesc import edu.uci.ics.texera.workflow.operators.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.texera.workflow.operators.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} +import edu.uci.ics.texera.workflow.operators.source.apis.twitter.v2.{ + TwitterFullArchiveSearchSourceOpDesc, + TwitterSearchSourceOpDesc +} import edu.uci.ics.texera.workflow.operators.source.fetcher.URLFetcherOpDesc import edu.uci.ics.texera.workflow.operators.source.scan.FileScanSourceOpDesc import edu.uci.ics.texera.workflow.operators.source.scan.csv.CSVScanSourceOpDesc @@ -53,7 +102,12 @@ import edu.uci.ics.texera.workflow.operators.symmetricDifference.SymmetricDiffer import edu.uci.ics.texera.workflow.operators.typecasting.TypeCastingOpDesc import edu.uci.ics.texera.workflow.operators.udf.java.JavaUDFOpDesc import edu.uci.ics.texera.workflow.operators.udf.python.source.PythonUDFSourceOpDescV2 -import edu.uci.ics.texera.workflow.operators.udf.python.{DualInputPortsPythonUDFOpDescV2, PythonLambdaFunctionOpDesc, PythonTableReducerOpDesc, PythonUDFOpDescV2} +import edu.uci.ics.texera.workflow.operators.udf.python.{ + DualInputPortsPythonUDFOpDescV2, + PythonLambdaFunctionOpDesc, + PythonTableReducerOpDesc, + PythonUDFOpDescV2 +} import edu.uci.ics.texera.workflow.operators.udf.r.{RUDFOpDesc, RUDFSourceOpDesc} import edu.uci.ics.texera.workflow.operators.union.UnionOpDesc import edu.uci.ics.texera.workflow.operators.unneststring.UnnestStringOpDesc diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala index ecc40c51f04..56b1f8e8ac5 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala @@ -49,13 +49,13 @@ class IfStatementOpDesc extends LogicalOp { OperatorGroupConstants.CONTROL_GROUP, inputPorts = List( InputPort(PortIdentity(), "Condition(State)"), - InputPort(PortIdentity(1), "Data", dependencies = List(PortIdentity()))), - outputPorts = List( - OutputPort(PortIdentity(), "True"), - OutputPort(PortIdentity(1), "False")) + InputPort(PortIdentity(1), "Data", dependencies = List(PortIdentity())) + ), + outputPorts = List(OutputPort(PortIdentity(), "True"), OutputPort(PortIdentity(1), "False")) ) override def getOutputSchema(schemas: Array[Schema]): Schema = throw new NotImplementedError() - override def getOutputSchemas(schemas: Array[Schema]): Array[Schema] = Array(schemas(1), schemas(1)) + override def getOutputSchemas(schemas: Array[Schema]): Array[Schema] = + Array(schemas(1), schemas(1)) } diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala index a9563d8d764..6af5b9caec1 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala @@ -13,7 +13,11 @@ class IfStatementOpExec(conditionName: String) extends OperatorExecutor { None } - override def processTupleMultiPort(tuple: Tuple, port: Int): Iterator[(TupleLike, Option[PortIdentity])] = Iterator((tuple, Some(PortIdentity(output_port)))) + override def processTupleMultiPort( + tuple: Tuple, + port: Int + ): Iterator[(TupleLike, Option[PortIdentity])] = + Iterator((tuple, Some(PortIdentity(output_port)))) override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = ??? } From b961031ee3330a32e5419be8caa2a2be4e7c025e Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 15:53:23 -0800 Subject: [PATCH 04/17] rename op --- .../uci/ics/texera/workflow/common/operators/LogicalOp.scala | 4 ++-- .../ifStatement/{IfStatementOpDesc.scala => IfOpDesc.scala} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/{IfStatementOpDesc.scala => IfOpDesc.scala} (98%) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala index 9218ed446ed..fade9c9edf9 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala @@ -22,7 +22,7 @@ import edu.uci.ics.texera.web.OPversion import edu.uci.ics.texera.workflow.common.metadata.{OperatorInfo, PropertyNameConstants} import edu.uci.ics.texera.workflow.operators.aggregate.AggregateOpDesc import edu.uci.ics.texera.workflow.operators.cartesianProduct.CartesianProductOpDesc -import edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement.IfStatementOpDesc +import edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement.IfOpDesc import edu.uci.ics.texera.workflow.operators.dictionary.DictionaryMatcherOpDesc import edu.uci.ics.texera.workflow.operators.difference.DifferenceOpDesc import edu.uci.ics.texera.workflow.operators.distinct.DistinctOpDesc @@ -160,7 +160,7 @@ trait StateTransferFunc ) @JsonSubTypes( Array( - new Type(value = classOf[IfStatementOpDesc], name = "IfStatement"), + new Type(value = classOf[IfOpDesc], name = "IfStatement"), new Type(value = classOf[SankeyDiagramOpDesc], name = "SankeyDiagram"), new Type(value = classOf[IcicleChartOpDesc], name = "IcicleChart"), new Type(value = classOf[CSVScanSourceOpDesc], name = "CSVFileScan"), diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala similarity index 98% rename from core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala rename to core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index 56b1f8e8ac5..99fe1696a8e 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -10,7 +10,7 @@ import edu.uci.ics.amber.engine.common.workflow.{InputPort, OutputPort, PortIden import edu.uci.ics.texera.workflow.common.metadata.{OperatorGroupConstants, OperatorInfo} import edu.uci.ics.texera.workflow.common.operators.LogicalOp -class IfStatementOpDesc extends LogicalOp { +class IfOpDesc extends LogicalOp { @JsonProperty(required = true) @JsonSchemaTitle("Condition State") @JsonPropertyDescription("name of the state variable to evaluate") From ce9a777cba93e82d81b326d9d89c65027e95555b Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 15:54:16 -0800 Subject: [PATCH 05/17] rename op --- .../workflow/operators/controlBlock/ifStatement/IfOpDesc.scala | 2 +- .../ifStatement/{IfStatementOpExec.scala => IfOpExec.scala} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/{IfStatementOpExec.scala => IfOpExec.scala} (91%) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index 99fe1696a8e..e16d20af07d 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -26,7 +26,7 @@ class IfOpDesc extends LogicalOp { executionId, operatorIdentifier, OpExecInitInfo((_, _) => { - new IfStatementOpExec(conditionName) + new IfOpExec(conditionName) }) ) .withInputPorts(operatorInfo.inputPorts) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala similarity index 91% rename from core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala rename to core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala index 6af5b9caec1..6daef24e50d 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfStatementOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala @@ -5,7 +5,7 @@ import edu.uci.ics.amber.engine.common.model.State import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} import edu.uci.ics.amber.engine.common.workflow.PortIdentity -class IfStatementOpExec(conditionName: String) extends OperatorExecutor { +class IfOpExec(conditionName: String) extends OperatorExecutor { private var output_port: Int = _ override def processState(state: State, port: Int): Option[State] = { From d0803356c136e7b3e3178eb41837b05d4b9b7fa8 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 15:55:21 -0800 Subject: [PATCH 06/17] rename op --- .../operators/controlBlock/ifStatement/IfOpDesc.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index e16d20af07d..e05a4a1055f 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -44,8 +44,8 @@ class IfOpDesc extends LogicalOp { override def operatorInfo: OperatorInfo = OperatorInfo( - "If Statement", - "If Statement", + "If", + "If", OperatorGroupConstants.CONTROL_GROUP, inputPorts = List( InputPort(PortIdentity(), "Condition(State)"), From 8ba8e95e73f62ada49e081035a583b20bd28aac2 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 16:14:27 -0800 Subject: [PATCH 07/17] Remove "Data" the name --- .../workflow/operators/controlBlock/ifStatement/IfOpDesc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index e05a4a1055f..10a7bbb3791 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -49,7 +49,7 @@ class IfOpDesc extends LogicalOp { OperatorGroupConstants.CONTROL_GROUP, inputPorts = List( InputPort(PortIdentity(), "Condition(State)"), - InputPort(PortIdentity(1), "Data", dependencies = List(PortIdentity())) + InputPort(PortIdentity(1), dependencies = List(PortIdentity())) ), outputPorts = List(OutputPort(PortIdentity(), "True"), OutputPort(PortIdentity(1), "False")) ) From f573953ba86bb52b05a46bb9edc20acf7d8e827a Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 16:14:56 -0800 Subject: [PATCH 08/17] Remove (State) in the port name --- .../workflow/operators/controlBlock/ifStatement/IfOpDesc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index 10a7bbb3791..95f0da23674 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -48,7 +48,7 @@ class IfOpDesc extends LogicalOp { "If", OperatorGroupConstants.CONTROL_GROUP, inputPorts = List( - InputPort(PortIdentity(), "Condition(State)"), + InputPort(PortIdentity(), "Condition"), InputPort(PortIdentity(1), dependencies = List(PortIdentity())) ), outputPorts = List(OutputPort(PortIdentity(), "True"), OutputPort(PortIdentity(1), "False")) From 09a4a0d1f0c8f63aab57d963b75430e15a7a1421 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 16:21:13 -0800 Subject: [PATCH 09/17] update --- .../workflow/operators/controlBlock/ifStatement/IfOpDesc.scala | 2 +- .../workflow/operators/controlBlock/ifStatement/IfOpExec.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index 95f0da23674..53552148794 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -51,7 +51,7 @@ class IfOpDesc extends LogicalOp { InputPort(PortIdentity(), "Condition"), InputPort(PortIdentity(1), dependencies = List(PortIdentity())) ), - outputPorts = List(OutputPort(PortIdentity(), "True"), OutputPort(PortIdentity(1), "False")) + outputPorts = List(OutputPort(PortIdentity(), "False"), OutputPort(PortIdentity(1), "True")) ) override def getOutputSchema(schemas: Array[Schema]): Schema = throw new NotImplementedError() diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala index 6daef24e50d..223ca9b54b1 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala @@ -9,7 +9,7 @@ class IfOpExec(conditionName: String) extends OperatorExecutor { private var output_port: Int = _ override def processState(state: State, port: Int): Option[State] = { - output_port = if (state.get(conditionName).asInstanceOf[Boolean]) 0 else 1 + output_port = if (state.get(conditionName).asInstanceOf[Boolean]) 1 else 0 None } From c44bb31fc2fdd994c57857099c78f9966da26e37 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 17:15:15 -0800 Subject: [PATCH 10/17] update --- .../operators/controlBlock/ifStatement/IfOpExec.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala index 223ca9b54b1..ca27add5e13 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala @@ -6,18 +6,21 @@ import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} import edu.uci.ics.amber.engine.common.workflow.PortIdentity class IfOpExec(conditionName: String) extends OperatorExecutor { - private var output_port: Int = _ + private var outputPort: PortIdentity = PortIdentity(1) // by default, it should be the true port. + //This function can handle one or more states. + //The state can have mutiple key-value pairs. Keys are not identified by conditionName will be ignored. + //It can accept any value that can be converted to a boolean. For example, Int 1 will be converted to true. override def processState(state: State, port: Int): Option[State] = { - output_port = if (state.get(conditionName).asInstanceOf[Boolean]) 1 else 0 - None + outputPort = if (state.get(conditionName).asInstanceOf[Boolean]) PortIdentity(1) else PortIdentity() + Some(state) } override def processTupleMultiPort( tuple: Tuple, port: Int ): Iterator[(TupleLike, Option[PortIdentity])] = - Iterator((tuple, Some(PortIdentity(output_port)))) + Iterator((tuple, Some(outputPort))) override def processTuple(tuple: Tuple, port: Int): Iterator[TupleLike] = ??? } From 3bb3d04c5bc2cbcdd9a49df7c898a7efcd846079 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Thu, 21 Nov 2024 21:38:09 -0800 Subject: [PATCH 11/17] fix fmt --- .../operators/controlBlock/ifStatement/IfOpExec.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala index ca27add5e13..0093e533a34 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala @@ -6,13 +6,14 @@ import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} import edu.uci.ics.amber.engine.common.workflow.PortIdentity class IfOpExec(conditionName: String) extends OperatorExecutor { - private var outputPort: PortIdentity = PortIdentity(1) // by default, it should be the true port. + private var outputPort: PortIdentity = PortIdentity(1) // by default, it should be the true port. //This function can handle one or more states. //The state can have mutiple key-value pairs. Keys are not identified by conditionName will be ignored. //It can accept any value that can be converted to a boolean. For example, Int 1 will be converted to true. override def processState(state: State, port: Int): Option[State] = { - outputPort = if (state.get(conditionName).asInstanceOf[Boolean]) PortIdentity(1) else PortIdentity() + outputPort = + if (state.get(conditionName).asInstanceOf[Boolean]) PortIdentity(1) else PortIdentity() Some(state) } From 3dfa0f9204b1154cbbcf179b48cfb54af45e9ca4 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 16 Dec 2024 21:03:12 -0800 Subject: [PATCH 12/17] update --- .../metadata/OperatorGroupConstants.scala | 53 --- .../workflow/common/operators/LogicalOp.scala | 409 ------------------ .../controlBlock/ifStatement/IfOpDesc.scala | 2 +- .../{IfStatement.png => If.png} | Bin .../uci/ics/amber/operator/LogicalOp.scala | 48 +- .../metadata/OperatorGroupConstants.scala | 4 +- 6 files changed, 9 insertions(+), 507 deletions(-) delete mode 100644 core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala delete mode 100644 core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala rename core/gui/src/assets/operator_images/{IfStatement.png => If.png} (100%) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala deleted file mode 100644 index 27aafccaa69..00000000000 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/metadata/OperatorGroupConstants.scala +++ /dev/null @@ -1,53 +0,0 @@ -package edu.uci.ics.texera.workflow.common.metadata - -object OperatorGroupConstants { - final val INPUT_GROUP = "Data Input" - final val DATABASE_GROUP = "Database Connector" - final val SEARCH_GROUP = "Search" - final val CLEANING_GROUP = "Data Cleaning" - final val JOIN_GROUP = "Join" - final val SET_GROUP = "Set" - final val AGGREGATE_GROUP = "Aggregate" - final val SORT_GROUP = "Sort" - final val UTILITY_GROUP = "Utilities" - final val API_GROUP = "External API" - final val VISUALIZATION_GROUP = "Visualization" - final val MACHINE_LEARNING_GROUP = "Machine Learning" - final val ADVANCED_SKLEARN_GROUP = "Advanced Sklearn" - final val HUGGINGFACE_GROUP = "Hugging Face" - final val SKLEARN_GROUP = "Sklearn" - final val UDF_GROUP = "User-defined Functions" - final val PYTHON_GROUP = "Python" - final val JAVA_GROUP = "Java" - final val R_GROUP = "R" - final val MACHINE_LEARNING_GENERAL_GROUP = "Machine Learning General" - final val CONTROL_GROUP = "Control Block" - - /** - * The order of the groups to show up in the frontend operator panel. - * The order numbers are relative. - */ - final val OperatorGroupOrderList: List[GroupInfo] = List( - GroupInfo(INPUT_GROUP), - GroupInfo(DATABASE_GROUP), - GroupInfo(SEARCH_GROUP), - GroupInfo( - CLEANING_GROUP, - List(GroupInfo(JOIN_GROUP), GroupInfo(AGGREGATE_GROUP), GroupInfo(SORT_GROUP)) - ), - GroupInfo( - MACHINE_LEARNING_GROUP, - List( - GroupInfo(SKLEARN_GROUP), - GroupInfo(ADVANCED_SKLEARN_GROUP), - GroupInfo(HUGGINGFACE_GROUP), - GroupInfo(MACHINE_LEARNING_GENERAL_GROUP) - ) - ), - GroupInfo(UTILITY_GROUP), - GroupInfo(API_GROUP), - GroupInfo(UDF_GROUP, List(GroupInfo(PYTHON_GROUP), GroupInfo(JAVA_GROUP), GroupInfo(R_GROUP))), - GroupInfo(VISUALIZATION_GROUP), - GroupInfo(CONTROL_GROUP) - ) -} diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala deleted file mode 100644 index fade9c9edf9..00000000000 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/common/operators/LogicalOp.scala +++ /dev/null @@ -1,409 +0,0 @@ -package edu.uci.ics.texera.workflow.common.operators - -import com.fasterxml.jackson.annotation.JsonSubTypes.Type -import com.fasterxml.jackson.annotation.{ - JsonIgnore, - JsonProperty, - JsonPropertyDescription, - JsonSubTypes, - JsonTypeInfo -} -import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle -import edu.uci.ics.amber.engine.common.executor.OperatorExecutor -import edu.uci.ics.amber.engine.common.model.{PhysicalOp, PhysicalPlan, WorkflowContext} -import edu.uci.ics.amber.engine.common.model.tuple.Schema -import edu.uci.ics.amber.engine.common.virtualidentity.{ - ExecutionIdentity, - OperatorIdentity, - WorkflowIdentity -} -import edu.uci.ics.amber.engine.common.workflow.PortIdentity -import edu.uci.ics.texera.web.OPversion -import edu.uci.ics.texera.workflow.common.metadata.{OperatorInfo, PropertyNameConstants} -import edu.uci.ics.texera.workflow.operators.aggregate.AggregateOpDesc -import edu.uci.ics.texera.workflow.operators.cartesianProduct.CartesianProductOpDesc -import edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement.IfOpDesc -import edu.uci.ics.texera.workflow.operators.dictionary.DictionaryMatcherOpDesc -import edu.uci.ics.texera.workflow.operators.difference.DifferenceOpDesc -import edu.uci.ics.texera.workflow.operators.distinct.DistinctOpDesc -import edu.uci.ics.texera.workflow.operators.download.BulkDownloaderOpDesc -import edu.uci.ics.texera.workflow.operators.dummy.DummyOpDesc -import edu.uci.ics.texera.workflow.operators.filter.SpecializedFilterOpDesc -import edu.uci.ics.texera.workflow.operators.hashJoin.HashJoinOpDesc -import edu.uci.ics.texera.workflow.operators.intersect.IntersectOpDesc -import edu.uci.ics.texera.workflow.operators.intervalJoin.IntervalJoinOpDesc -import edu.uci.ics.texera.workflow.operators.keywordSearch.KeywordSearchOpDesc -import edu.uci.ics.texera.workflow.operators.limit.LimitOpDesc -import edu.uci.ics.texera.workflow.operators.huggingFace.{ - HuggingFaceIrisLogisticRegressionOpDesc, - HuggingFaceSentimentAnalysisOpDesc, - HuggingFaceSpamSMSDetectionOpDesc, - HuggingFaceTextSummarizationOpDesc -} -import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.KNNTrainer.{ - SklearnAdvancedKNNClassifierTrainerOpDesc, - SklearnAdvancedKNNRegressorTrainerOpDesc -} -import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc -import edu.uci.ics.texera.workflow.operators.machineLearning.sklearnAdvanced.SVRTrainer.SVCTrainer.SklearnAdvancedSVRTrainerOpDesc -import edu.uci.ics.texera.workflow.operators.projection.ProjectionOpDesc -import edu.uci.ics.texera.workflow.operators.randomksampling.RandomKSamplingOpDesc -import edu.uci.ics.texera.workflow.operators.regex.RegexOpDesc -import edu.uci.ics.texera.workflow.operators.reservoirsampling.ReservoirSamplingOpDesc -import edu.uci.ics.texera.workflow.operators.sentiment.SentimentAnalysisOpDesc -import edu.uci.ics.texera.workflow.operators.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.texera.workflow.operators.sklearn.{ - SklearnAdaptiveBoostingOpDesc, - SklearnBaggingOpDesc, - SklearnBernoulliNaiveBayesOpDesc, - SklearnComplementNaiveBayesOpDesc, - SklearnDecisionTreeOpDesc, - SklearnDummyClassifierOpDesc, - SklearnExtraTreeOpDesc, - SklearnExtraTreesOpDesc, - SklearnGaussianNaiveBayesOpDesc, - SklearnGradientBoostingOpDesc, - SklearnKNNOpDesc, - SklearnLinearRegressionOpDesc, - SklearnLinearSVMOpDesc, - SklearnLogisticRegressionCVOpDesc, - SklearnLogisticRegressionOpDesc, - SklearnMultiLayerPerceptronOpDesc, - SklearnMultinomialNaiveBayesOpDesc, - SklearnNearestCentroidOpDesc, - SklearnPassiveAggressiveOpDesc, - SklearnPerceptronOpDesc, - SklearnPredictionOpDesc, - SklearnProbabilityCalibrationOpDesc, - SklearnRandomForestOpDesc, - SklearnRidgeCVOpDesc, - SklearnRidgeOpDesc, - SklearnSDGOpDesc, - SklearnSVMOpDesc -} -import edu.uci.ics.texera.workflow.operators.sort.SortOpDesc -import edu.uci.ics.texera.workflow.operators.sortPartitions.SortPartitionsOpDesc -import edu.uci.ics.texera.workflow.operators.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.apis.twitter.v2.{ - TwitterFullArchiveSearchSourceOpDesc, - TwitterSearchSourceOpDesc -} -import edu.uci.ics.texera.workflow.operators.source.fetcher.URLFetcherOpDesc -import edu.uci.ics.texera.workflow.operators.source.scan.FileScanSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.scan.csv.CSVScanSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.scan.csvOld.CSVOldScanSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.scan.json.JSONLScanSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.scan.text.TextInputSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.sql.asterixdb.AsterixDBSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.sql.mysql.MySQLSourceOpDesc -import edu.uci.ics.texera.workflow.operators.source.sql.postgresql.PostgreSQLSourceOpDesc -import edu.uci.ics.texera.workflow.operators.split.SplitOpDesc -import edu.uci.ics.texera.workflow.operators.symmetricDifference.SymmetricDifferenceOpDesc -import edu.uci.ics.texera.workflow.operators.typecasting.TypeCastingOpDesc -import edu.uci.ics.texera.workflow.operators.udf.java.JavaUDFOpDesc -import edu.uci.ics.texera.workflow.operators.udf.python.source.PythonUDFSourceOpDescV2 -import edu.uci.ics.texera.workflow.operators.udf.python.{ - DualInputPortsPythonUDFOpDescV2, - PythonLambdaFunctionOpDesc, - PythonTableReducerOpDesc, - PythonUDFOpDescV2 -} -import edu.uci.ics.texera.workflow.operators.udf.r.{RUDFOpDesc, RUDFSourceOpDesc} -import edu.uci.ics.texera.workflow.operators.union.UnionOpDesc -import edu.uci.ics.texera.workflow.operators.unneststring.UnnestStringOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.candlestickChart.CandlestickChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.boxPlot.BoxPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.barChart.BarChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.htmlviz.HtmlVizOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.pieChart.PieChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.scatterplot.ScatterplotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.ganttChart.GanttChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.urlviz.UrlVizOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.DotPlot.DotPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.wordCloud.WordCloudOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.filledAreaPlot.FilledAreaPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.bubbleChart.BubbleChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.ImageViz.ImageVisualizerOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.hierarchychart.HierarchyChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.dumbbellPlot.DumbbellPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.histogram.HistogramChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.heatMap.HeatMapOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.lineChart.LineChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.waterfallChart.WaterfallChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.scatter3DChart.Scatter3dChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.ScatterMatrixChart.ScatterMatrixChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.funnelPlot.FunnelPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.tablesChart.TablesPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.icicleChart.IcicleChartOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.continuousErrorBands.ContinuousErrorBandsOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.ternaryPlot.TernaryPlotOpDesc -import org.apache.commons.lang3.builder.{EqualsBuilder, HashCodeBuilder, ToStringBuilder} -import org.apache.zookeeper.KeeperException.UnimplementedException -import edu.uci.ics.texera.workflow.operators.machineLearning.Scorer.MachineLearningScorerOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.quiverPlot.QuiverPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.contourPlot.ContourPlotOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.figureFactoryTable.FigureFactoryTableOpDesc -import edu.uci.ics.texera.workflow.operators.visualization.sankeyDiagram.SankeyDiagramOpDesc - -import java.util.UUID -import scala.collection.mutable -import scala.util.Try - -trait StateTransferFunc - extends ((OperatorExecutor, OperatorExecutor) => Unit) - with java.io.Serializable - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "operatorType" -) -@JsonSubTypes( - Array( - new Type(value = classOf[IfOpDesc], name = "IfStatement"), - new Type(value = classOf[SankeyDiagramOpDesc], name = "SankeyDiagram"), - new Type(value = classOf[IcicleChartOpDesc], name = "IcicleChart"), - new Type(value = classOf[CSVScanSourceOpDesc], name = "CSVFileScan"), - // disabled the ParallelCSVScanSourceOpDesc so that it does not confuse user. it can be re-enabled when doing experiments. - // new Type(value = classOf[ParallelCSVScanSourceOpDesc], name = "ParallelCSVFileScan"), - new Type(value = classOf[JSONLScanSourceOpDesc], name = "JSONLFileScan"), - new Type(value = classOf[FileScanSourceOpDesc], name = "FileScan"), - new Type(value = classOf[TextInputSourceOpDesc], name = "TextInput"), - new Type( - value = classOf[TwitterFullArchiveSearchSourceOpDesc], - name = "TwitterFullArchiveSearch" - ), - new Type( - value = classOf[TwitterSearchSourceOpDesc], - name = "TwitterSearch" - ), - new Type(value = classOf[ProgressiveSinkOpDesc], name = "SimpleSink"), - new Type(value = classOf[CandlestickChartOpDesc], name = "CandlestickChart"), - new Type(value = classOf[SplitOpDesc], name = "Split"), - new Type(value = classOf[ContourPlotOpDesc], name = "ContourPlot"), - new Type(value = classOf[RegexOpDesc], name = "Regex"), - new Type(value = classOf[SpecializedFilterOpDesc], name = "Filter"), - new Type(value = classOf[SentimentAnalysisOpDesc], name = "SentimentAnalysis"), - new Type(value = classOf[ProjectionOpDesc], name = "Projection"), - new Type(value = classOf[UnionOpDesc], name = "Union"), - new Type(value = classOf[KeywordSearchOpDesc], name = "KeywordSearch"), - new Type(value = classOf[AggregateOpDesc], name = "Aggregate"), - new Type(value = classOf[LineChartOpDesc], name = "LineChart"), - new Type(value = classOf[WaterfallChartOpDesc], name = "WaterfallChart"), - new Type(value = classOf[BarChartOpDesc], name = "BarChart"), - new Type(value = classOf[PieChartOpDesc], name = "PieChart"), - new Type(value = classOf[QuiverPlotOpDesc], name = "QuiverPlot"), - new Type(value = classOf[WordCloudOpDesc], name = "WordCloud"), - new Type(value = classOf[HtmlVizOpDesc], name = "HTMLVisualizer"), - new Type(value = classOf[UrlVizOpDesc], name = "URLVisualizer"), - new Type(value = classOf[ScatterplotOpDesc], name = "Scatterplot"), - new Type(value = classOf[PythonUDFOpDescV2], name = "PythonUDFV2"), - new Type(value = classOf[PythonUDFSourceOpDescV2], name = "PythonUDFSourceV2"), - new Type(value = classOf[DualInputPortsPythonUDFOpDescV2], name = "DualInputPortsPythonUDFV2"), - new Type(value = classOf[MySQLSourceOpDesc], name = "MySQLSource"), - new Type(value = classOf[PostgreSQLSourceOpDesc], name = "PostgreSQLSource"), - new Type(value = classOf[AsterixDBSourceOpDesc], name = "AsterixDBSource"), - new Type(value = classOf[TypeCastingOpDesc], name = "TypeCasting"), - new Type(value = classOf[LimitOpDesc], name = "Limit"), - new Type(value = classOf[RandomKSamplingOpDesc], name = "RandomKSampling"), - new Type(value = classOf[ReservoirSamplingOpDesc], name = "ReservoirSampling"), - new Type(value = classOf[HashJoinOpDesc[String]], name = "HashJoin"), - new Type(value = classOf[DistinctOpDesc], name = "Distinct"), - new Type(value = classOf[IntersectOpDesc], name = "Intersect"), - new Type(value = classOf[SymmetricDifferenceOpDesc], name = "SymmetricDifference"), - new Type(value = classOf[DifferenceOpDesc], name = "Difference"), - new Type(value = classOf[IntervalJoinOpDesc], name = "IntervalJoin"), - new Type(value = classOf[UnnestStringOpDesc], name = "UnnestString"), - new Type(value = classOf[DictionaryMatcherOpDesc], name = "DictionaryMatcher"), - new Type(value = classOf[SortPartitionsOpDesc], name = "SortPartitions"), - new Type(value = classOf[CSVOldScanSourceOpDesc], name = "CSVOldFileScan"), - new Type(value = classOf[RedditSearchSourceOpDesc], name = "RedditSearch"), - new Type(value = classOf[PythonLambdaFunctionOpDesc], name = "PythonLambdaFunction"), - new Type(value = classOf[PythonTableReducerOpDesc], name = "PythonTableReducer"), - new Type(value = classOf[BulkDownloaderOpDesc], name = "BulkDownloader"), - new Type(value = classOf[URLFetcherOpDesc], name = "URLFetcher"), - new Type(value = classOf[CartesianProductOpDesc], name = "CartesianProduct"), - new Type(value = classOf[FilledAreaPlotOpDesc], name = "FilledAreaPlot"), - new Type(value = classOf[DotPlotOpDesc], name = "DotPlot"), - new Type(value = classOf[BubbleChartOpDesc], name = "BubbleChart"), - new Type(value = classOf[GanttChartOpDesc], name = "GanttChart"), - new Type(value = classOf[ImageVisualizerOpDesc], name = "ImageVisualizer"), - new Type(value = classOf[HierarchyChartOpDesc], name = "HierarchyChart"), - new Type(value = classOf[DumbbellPlotOpDesc], name = "DumbbellPlot"), - new Type(value = classOf[DummyOpDesc], name = "Dummy"), - new Type(value = classOf[BoxPlotOpDesc], name = "BoxPlot"), - new Type(value = classOf[HistogramChartOpDesc], name = "Histogram"), - new Type(value = classOf[ScatterMatrixChartOpDesc], name = "ScatterMatrixChart"), - new Type(value = classOf[HeatMapOpDesc], name = "HeatMap"), - new Type(value = classOf[Scatter3dChartOpDesc], name = "Scatter3DChart"), - new Type(value = classOf[FunnelPlotOpDesc], name = "FunnelPlot"), - new Type(value = classOf[TablesPlotOpDesc], name = "TablesPlot"), - new Type(value = classOf[ContinuousErrorBandsOpDesc], name = "ContinuousErrorBands"), - new Type(value = classOf[FigureFactoryTableOpDesc], name = "FigureFactoryTable"), - new Type(value = classOf[TernaryPlotOpDesc], name = "TernaryPlot"), - new Type(value = classOf[JavaUDFOpDesc], name = "JavaUDF"), - new Type(value = classOf[RUDFOpDesc], name = "RUDF"), - new Type(value = classOf[RUDFSourceOpDesc], name = "RUDFSource"), - new Type(value = classOf[MachineLearningScorerOpDesc], name = "Scorer"), - new Type(value = classOf[SortOpDesc], name = "Sort"), - new Type(value = classOf[SklearnLogisticRegressionOpDesc], name = "SklearnLogisticRegression"), - new Type( - value = classOf[SklearnLogisticRegressionCVOpDesc], - name = "SklearnLogisticRegressionCV" - ), - new Type(value = classOf[SklearnRidgeOpDesc], name = "SklearnRidge"), - new Type(value = classOf[SklearnRidgeCVOpDesc], name = "SklearnRidgeCV"), - new Type(value = classOf[SklearnSDGOpDesc], name = "SklearnSDG"), - new Type(value = classOf[SklearnPassiveAggressiveOpDesc], name = "SklearnPassiveAggressive"), - new Type(value = classOf[SklearnPerceptronOpDesc], name = "SklearnPerceptron"), - new Type(value = classOf[SklearnKNNOpDesc], name = "SklearnKNN"), - new Type(value = classOf[SklearnNearestCentroidOpDesc], name = "SklearnNearestCentroid"), - new Type(value = classOf[SklearnSVMOpDesc], name = "SklearnSVM"), - new Type(value = classOf[SklearnLinearSVMOpDesc], name = "SklearnLinearSVM"), - new Type(value = classOf[SklearnLinearRegressionOpDesc], name = "SklearnLinearRegression"), - new Type(value = classOf[SklearnDecisionTreeOpDesc], name = "SklearnDecisionTree"), - new Type(value = classOf[SklearnExtraTreeOpDesc], name = "SklearnExtraTree"), - new Type( - value = classOf[SklearnMultiLayerPerceptronOpDesc], - name = "SklearnMultiLayerPerceptron" - ), - new Type( - value = classOf[SklearnProbabilityCalibrationOpDesc], - name = "SklearnProbabilityCalibration" - ), - new Type(value = classOf[SklearnRandomForestOpDesc], name = "SklearnRandomForest"), - new Type(value = classOf[SklearnBaggingOpDesc], name = "SklearnBagging"), - new Type(value = classOf[SklearnGradientBoostingOpDesc], name = "SklearnGradientBoosting"), - new Type(value = classOf[SklearnAdaptiveBoostingOpDesc], name = "SklearnAdaptiveBoosting"), - new Type(value = classOf[SklearnExtraTreesOpDesc], name = "SklearnExtraTrees"), - new Type(value = classOf[SklearnGaussianNaiveBayesOpDesc], name = "SklearnGaussianNaiveBayes"), - new Type( - value = classOf[SklearnMultinomialNaiveBayesOpDesc], - name = "SklearnMultinomialNaiveBayes" - ), - new Type( - value = classOf[SklearnComplementNaiveBayesOpDesc], - name = "SklearnComplementNaiveBayes" - ), - new Type( - value = classOf[SklearnBernoulliNaiveBayesOpDesc], - name = "SklearnBernoulliNaiveBayes" - ), - new Type(value = classOf[SklearnDummyClassifierOpDesc], name = "SklearnDummyClassifier"), - new Type(value = classOf[SklearnPredictionOpDesc], name = "SklearnPrediction"), - new Type( - value = classOf[HuggingFaceSentimentAnalysisOpDesc], - name = "HuggingFaceSentimentAnalysis" - ), - new Type( - value = classOf[HuggingFaceTextSummarizationOpDesc], - name = "HuggingFaceTextSummarization" - ), - new Type( - value = classOf[HuggingFaceSpamSMSDetectionOpDesc], - name = "HuggingFaceSpamSMSDetection" - ), - new Type( - value = classOf[HuggingFaceIrisLogisticRegressionOpDesc], - name = "HuggingFaceIrisLogisticRegression" - ), - new Type( - value = classOf[SklearnAdvancedKNNClassifierTrainerOpDesc], - name = "KNNClassifierTrainer" - ), - new Type( - value = classOf[SklearnAdvancedKNNRegressorTrainerOpDesc], - name = "KNNRegressorTrainer" - ), - new Type( - value = classOf[SklearnAdvancedSVCTrainerOpDesc], - name = "SVCTrainer" - ), - new Type( - value = classOf[SklearnAdvancedSVRTrainerOpDesc], - name = "SVRTrainer" - ) - ) -) -abstract class LogicalOp extends PortDescriptor with Serializable { - - @JsonIgnore - private var context: WorkflowContext = _ - - @JsonProperty(PropertyNameConstants.OPERATOR_ID) - private var operatorId: String = getClass.getSimpleName + "-" + UUID.randomUUID.toString - - @JsonProperty(PropertyNameConstants.OPERATOR_VERSION) - var operatorVersion: String = getOperatorVersion() - - @JsonIgnore - val inputPortToSchemaMapping: mutable.Map[PortIdentity, Schema] = mutable.HashMap() - @JsonIgnore - val outputPortToSchemaMapping: mutable.Map[PortIdentity, Schema] = mutable.HashMap() - def operatorIdentifier: OperatorIdentity = OperatorIdentity(operatorId) - - def getPhysicalOp( - workflowId: WorkflowIdentity, - executionId: ExecutionIdentity - ): PhysicalOp = { - throw new UnimplementedException() - } - - // a logical operator corresponds multiple physical operators (a small DAG) - def getPhysicalPlan( - workflowId: WorkflowIdentity, - executionId: ExecutionIdentity - ): PhysicalPlan = { - new PhysicalPlan( - operators = Set(getPhysicalOp(workflowId, executionId)), - links = Set.empty - ) - } - - def operatorInfo: OperatorInfo - - def getOutputSchema(schemas: Array[Schema]): Schema - - private def getOperatorVersion(): String = { - val path = "core/amber/src/main/scala/" - val operatorPath = path + this.getClass.getPackage.getName.replace(".", "/") - OPversion.getVersion(this.getClass.getSimpleName, operatorPath) - } - - // override if the operator has multiple output ports, schema must be specified for each port - def getOutputSchemas(schemas: Array[Schema]): Array[Schema] = { - Array.fill(1)(getOutputSchema(schemas)) - } - - override def hashCode: Int = HashCodeBuilder.reflectionHashCode(this) - - override def equals(that: Any): Boolean = EqualsBuilder.reflectionEquals(this, that, "context") - - override def toString: String = ToStringBuilder.reflectionToString(this) - - def getContext: WorkflowContext = this.context - def setContext(workflowContext: WorkflowContext): Unit = { - this.context = workflowContext - } - - def setOperatorId(id: String): Unit = { - operatorId = id - } - - def runtimeReconfiguration( - workflowId: WorkflowIdentity, - executionId: ExecutionIdentity, - oldOpDesc: LogicalOp, - newOpDesc: LogicalOp - ): Try[(PhysicalOp, Option[StateTransferFunc])] = { - throw new UnsupportedOperationException( - "operator " + getClass.getSimpleName + " does not support reconfiguration" - ) - } - - @JsonProperty - @JsonSchemaTitle("Dummy Property List") - @JsonPropertyDescription("Add dummy property if needed") - var dummyPropertyList: List[DummyProperties] = List() - -} diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala index 53552148794..96a9b42aeab 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala @@ -7,7 +7,7 @@ import edu.uci.ics.amber.engine.common.model.{PhysicalOp, SchemaPropagationFunc} import edu.uci.ics.amber.engine.common.model.tuple.Schema import edu.uci.ics.amber.engine.common.virtualidentity.{ExecutionIdentity, WorkflowIdentity} import edu.uci.ics.amber.engine.common.workflow.{InputPort, OutputPort, PortIdentity} -import edu.uci.ics.texera.workflow.common.metadata.{OperatorGroupConstants, OperatorInfo} +import edu.uci.ics.texera.workflow.common.metadata.OperatorInfo import edu.uci.ics.texera.workflow.common.operators.LogicalOp class IfOpDesc extends LogicalOp { diff --git a/core/gui/src/assets/operator_images/IfStatement.png b/core/gui/src/assets/operator_images/If.png similarity index 100% rename from core/gui/src/assets/operator_images/IfStatement.png rename to core/gui/src/assets/operator_images/If.png diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index ac88f098cd3..cf909ef7338 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -14,21 +14,13 @@ import edu.uci.ics.amber.operator.distinct.DistinctOpDesc import edu.uci.ics.amber.operator.dummy.DummyOpDesc import edu.uci.ics.amber.operator.filter.SpecializedFilterOpDesc import edu.uci.ics.amber.operator.hashJoin.HashJoinOpDesc -import edu.uci.ics.amber.operator.huggingFace.{ - HuggingFaceIrisLogisticRegressionOpDesc, - HuggingFaceSentimentAnalysisOpDesc, - HuggingFaceSpamSMSDetectionOpDesc, - HuggingFaceTextSummarizationOpDesc -} +import edu.uci.ics.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} import edu.uci.ics.amber.operator.intersect.IntersectOpDesc import edu.uci.ics.amber.operator.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.amber.operator.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.amber.operator.limit.LimitOpDesc import edu.uci.ics.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ - SklearnAdvancedKNNClassifierTrainerOpDesc, - SklearnAdvancedKNNRegressorTrainerOpDesc -} +import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -38,42 +30,11 @@ import edu.uci.ics.amber.operator.regex.RegexOpDesc import edu.uci.ics.amber.operator.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.amber.operator.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.amber.operator.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.amber.operator.sklearn.{ - SklearnAdaptiveBoostingOpDesc, - SklearnBaggingOpDesc, - SklearnBernoulliNaiveBayesOpDesc, - SklearnComplementNaiveBayesOpDesc, - SklearnDecisionTreeOpDesc, - SklearnDummyClassifierOpDesc, - SklearnExtraTreeOpDesc, - SklearnExtraTreesOpDesc, - SklearnGaussianNaiveBayesOpDesc, - SklearnGradientBoostingOpDesc, - SklearnKNNOpDesc, - SklearnLinearRegressionOpDesc, - SklearnLinearSVMOpDesc, - SklearnLogisticRegressionCVOpDesc, - SklearnLogisticRegressionOpDesc, - SklearnMultiLayerPerceptronOpDesc, - SklearnMultinomialNaiveBayesOpDesc, - SklearnNearestCentroidOpDesc, - SklearnPassiveAggressiveOpDesc, - SklearnPerceptronOpDesc, - SklearnPredictionOpDesc, - SklearnProbabilityCalibrationOpDesc, - SklearnRandomForestOpDesc, - SklearnRidgeCVOpDesc, - SklearnRidgeOpDesc, - SklearnSDGOpDesc, - SklearnSVMOpDesc -} +import edu.uci.ics.amber.operator.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} import edu.uci.ics.amber.operator.sort.SortOpDesc import edu.uci.ics.amber.operator.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.amber.operator.source.apis.twitter.v2.{ - TwitterFullArchiveSearchSourceOpDesc, - TwitterSearchSourceOpDesc -} +import edu.uci.ics.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} import edu.uci.ics.amber.operator.source.fetcher.URLFetcherOpDesc import edu.uci.ics.amber.operator.source.scan.FileScanSourceOpDesc import edu.uci.ics.amber.operator.source.scan.csv.CSVScanSourceOpDesc @@ -142,6 +103,7 @@ trait StateTransferFunc ) @JsonSubTypes( Array( + new Type(value = classOf[IfOpDesc], name = "If"), new Type(value = classOf[SankeyDiagramOpDesc], name = "SankeyDiagram"), new Type(value = classOf[IcicleChartOpDesc], name = "IcicleChart"), new Type(value = classOf[CSVScanSourceOpDesc], name = "CSVFileScan"), diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/metadata/OperatorGroupConstants.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/metadata/OperatorGroupConstants.scala index 9ea299a55d4..4bfb59f3047 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/metadata/OperatorGroupConstants.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/metadata/OperatorGroupConstants.scala @@ -21,6 +21,7 @@ object OperatorGroupConstants { final val JAVA_GROUP = "Java" final val R_GROUP = "R" final val MACHINE_LEARNING_GENERAL_GROUP = "Machine Learning General" + final val CONTROL_GROUP = "Control Block" /** * The order of the groups to show up in the frontend operator panel. @@ -46,6 +47,7 @@ object OperatorGroupConstants { GroupInfo(UTILITY_GROUP), GroupInfo(API_GROUP), GroupInfo(UDF_GROUP, List(GroupInfo(PYTHON_GROUP), GroupInfo(JAVA_GROUP), GroupInfo(R_GROUP))), - GroupInfo(VISUALIZATION_GROUP) + GroupInfo(VISUALIZATION_GROUP), + GroupInfo(CONTROL_GROUP) ) } From 58b16502fe433b0a4cb63c4baaee9baea59ae048 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 16 Dec 2024 21:04:13 -0800 Subject: [PATCH 13/17] update --- .../main/scala/edu/uci/ics/amber/operator/LogicalOp.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index cf909ef7338..6853bcc5894 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -14,7 +14,12 @@ import edu.uci.ics.amber.operator.distinct.DistinctOpDesc import edu.uci.ics.amber.operator.dummy.DummyOpDesc import edu.uci.ics.amber.operator.filter.SpecializedFilterOpDesc import edu.uci.ics.amber.operator.hashJoin.HashJoinOpDesc -import edu.uci.ics.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} +import edu.uci.ics.amber.operator.huggingFace.{ + HuggingFaceIrisLogisticRegressionOpDesc, + HuggingFaceSentimentAnalysisOpDesc, + HuggingFaceSpamSMSDetectionOpDesc, + HuggingFaceTextSummarizationOpDesc +} import edu.uci.ics.amber.operator.intersect.IntersectOpDesc import edu.uci.ics.amber.operator.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.amber.operator.keywordSearch.KeywordSearchOpDesc From 412e7288d00592b7bfd7380c4254f1ae920bc6ec Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 16 Dec 2024 21:05:14 -0800 Subject: [PATCH 14/17] update --- .../uci/ics/amber/operator/LogicalOp.scala | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index 6853bcc5894..9e2f9018a97 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -25,7 +25,10 @@ import edu.uci.ics.amber.operator.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.amber.operator.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.amber.operator.limit.LimitOpDesc import edu.uci.ics.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} +import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ + SklearnAdvancedKNNClassifierTrainerOpDesc, + SklearnAdvancedKNNRegressorTrainerOpDesc +} import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -35,7 +38,35 @@ import edu.uci.ics.amber.operator.regex.RegexOpDesc import edu.uci.ics.amber.operator.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.amber.operator.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.amber.operator.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.amber.operator.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} +import edu.uci.ics.amber.operator.sklearn.{ + SklearnAdaptiveBoostingOpDesc, + SklearnBaggingOpDesc, + SklearnBernoulliNaiveBayesOpDesc, + SklearnComplementNaiveBayesOpDesc, + SklearnDecisionTreeOpDesc, + SklearnDummyClassifierOpDesc, + SklearnExtraTreeOpDesc, + SklearnExtraTreesOpDesc, + SklearnGaussianNaiveBayesOpDesc, + SklearnGradientBoostingOpDesc, + SklearnKNNOpDesc, + SklearnLinearRegressionOpDesc, + SklearnLinearSVMOpDesc, + SklearnLogisticRegressionCVOpDesc, + SklearnLogisticRegressionOpDesc, + SklearnMultiLayerPerceptronOpDesc, + SklearnMultinomialNaiveBayesOpDesc, + SklearnNearestCentroidOpDesc, + SklearnPassiveAggressiveOpDesc, + SklearnPerceptronOpDesc, + SklearnPredictionOpDesc, + SklearnProbabilityCalibrationOpDesc, + SklearnRandomForestOpDesc, + SklearnRidgeCVOpDesc, + SklearnRidgeOpDesc, + SklearnSDGOpDesc, + SklearnSVMOpDesc +} import edu.uci.ics.amber.operator.sort.SortOpDesc import edu.uci.ics.amber.operator.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc From adda4d6d044adf82320c7967b6486e59632aaac3 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 16 Dec 2024 21:05:41 -0800 Subject: [PATCH 15/17] update --- .../main/scala/edu/uci/ics/amber/operator/LogicalOp.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index 9e2f9018a97..1d1c49a6f16 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -70,7 +70,10 @@ import edu.uci.ics.amber.operator.sklearn.{ import edu.uci.ics.amber.operator.sort.SortOpDesc import edu.uci.ics.amber.operator.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} +import edu.uci.ics.amber.operator.source.apis.twitter.v2.{ + TwitterFullArchiveSearchSourceOpDesc, + TwitterSearchSourceOpDesc +} import edu.uci.ics.amber.operator.source.fetcher.URLFetcherOpDesc import edu.uci.ics.amber.operator.source.scan.FileScanSourceOpDesc import edu.uci.ics.amber.operator.source.scan.csv.CSVScanSourceOpDesc From e5a7a5238ab8f5e196a3b280d0f3d266df5051cc Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Mon, 16 Dec 2024 23:45:25 -0800 Subject: [PATCH 16/17] update --- .../uci/ics/amber/operator/LogicalOp.scala | 48 ++----------------- .../operator}/ifStatement/IfOpDesc.scala | 16 +++---- .../operator}/ifStatement/IfOpExec.scala | 10 ++-- 3 files changed, 18 insertions(+), 56 deletions(-) rename core/{amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock => workflow-operator/src/main/scala/edu/uci/ics/amber/operator}/ifStatement/IfOpDesc.scala (73%) rename core/{amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock => workflow-operator/src/main/scala/edu/uci/ics/amber/operator}/ifStatement/IfOpExec.scala (73%) diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index 1d1c49a6f16..1d2551b4f4b 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -14,21 +14,14 @@ import edu.uci.ics.amber.operator.distinct.DistinctOpDesc import edu.uci.ics.amber.operator.dummy.DummyOpDesc import edu.uci.ics.amber.operator.filter.SpecializedFilterOpDesc import edu.uci.ics.amber.operator.hashJoin.HashJoinOpDesc -import edu.uci.ics.amber.operator.huggingFace.{ - HuggingFaceIrisLogisticRegressionOpDesc, - HuggingFaceSentimentAnalysisOpDesc, - HuggingFaceSpamSMSDetectionOpDesc, - HuggingFaceTextSummarizationOpDesc -} +import edu.uci.ics.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} +import edu.uci.ics.amber.operator.ifStatement.IfOpDesc import edu.uci.ics.amber.operator.intersect.IntersectOpDesc import edu.uci.ics.amber.operator.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.amber.operator.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.amber.operator.limit.LimitOpDesc import edu.uci.ics.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ - SklearnAdvancedKNNClassifierTrainerOpDesc, - SklearnAdvancedKNNRegressorTrainerOpDesc -} +import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -38,42 +31,11 @@ import edu.uci.ics.amber.operator.regex.RegexOpDesc import edu.uci.ics.amber.operator.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.amber.operator.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.amber.operator.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.amber.operator.sklearn.{ - SklearnAdaptiveBoostingOpDesc, - SklearnBaggingOpDesc, - SklearnBernoulliNaiveBayesOpDesc, - SklearnComplementNaiveBayesOpDesc, - SklearnDecisionTreeOpDesc, - SklearnDummyClassifierOpDesc, - SklearnExtraTreeOpDesc, - SklearnExtraTreesOpDesc, - SklearnGaussianNaiveBayesOpDesc, - SklearnGradientBoostingOpDesc, - SklearnKNNOpDesc, - SklearnLinearRegressionOpDesc, - SklearnLinearSVMOpDesc, - SklearnLogisticRegressionCVOpDesc, - SklearnLogisticRegressionOpDesc, - SklearnMultiLayerPerceptronOpDesc, - SklearnMultinomialNaiveBayesOpDesc, - SklearnNearestCentroidOpDesc, - SklearnPassiveAggressiveOpDesc, - SklearnPerceptronOpDesc, - SklearnPredictionOpDesc, - SklearnProbabilityCalibrationOpDesc, - SklearnRandomForestOpDesc, - SklearnRidgeCVOpDesc, - SklearnRidgeOpDesc, - SklearnSDGOpDesc, - SklearnSVMOpDesc -} +import edu.uci.ics.amber.operator.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} import edu.uci.ics.amber.operator.sort.SortOpDesc import edu.uci.ics.amber.operator.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.amber.operator.source.apis.twitter.v2.{ - TwitterFullArchiveSearchSourceOpDesc, - TwitterSearchSourceOpDesc -} +import edu.uci.ics.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} import edu.uci.ics.amber.operator.source.fetcher.URLFetcherOpDesc import edu.uci.ics.amber.operator.source.scan.FileScanSourceOpDesc import edu.uci.ics.amber.operator.source.scan.csv.CSVScanSourceOpDesc diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpDesc.scala similarity index 73% rename from core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala rename to core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpDesc.scala index 96a9b42aeab..f4c0d7db2b6 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpDesc.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpDesc.scala @@ -1,14 +1,14 @@ -package edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement +package edu.uci.ics.amber.operator.ifStatement import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription} import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle -import edu.uci.ics.amber.engine.architecture.deploysemantics.layer.OpExecInitInfo -import edu.uci.ics.amber.engine.common.model.{PhysicalOp, SchemaPropagationFunc} -import edu.uci.ics.amber.engine.common.model.tuple.Schema -import edu.uci.ics.amber.engine.common.virtualidentity.{ExecutionIdentity, WorkflowIdentity} -import edu.uci.ics.amber.engine.common.workflow.{InputPort, OutputPort, PortIdentity} -import edu.uci.ics.texera.workflow.common.metadata.OperatorInfo -import edu.uci.ics.texera.workflow.common.operators.LogicalOp +import edu.uci.ics.amber.core.executor.OpExecInitInfo +import edu.uci.ics.amber.core.tuple.Schema +import edu.uci.ics.amber.core.workflow.{PhysicalOp, SchemaPropagationFunc} +import edu.uci.ics.amber.operator.LogicalOp +import edu.uci.ics.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} +import edu.uci.ics.amber.virtualidentity.{ExecutionIdentity, WorkflowIdentity} +import edu.uci.ics.amber.workflow.{InputPort, OutputPort, PortIdentity} class IfOpDesc extends LogicalOp { @JsonProperty(required = true) diff --git a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpExec.scala similarity index 73% rename from core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala rename to core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpExec.scala index 0093e533a34..c96bd7457d3 100644 --- a/core/amber/src/main/scala/edu/uci/ics/texera/workflow/operators/controlBlock/ifStatement/IfOpExec.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/ifStatement/IfOpExec.scala @@ -1,9 +1,9 @@ -package edu.uci.ics.texera.workflow.operators.controlBlock.ifStatement +package edu.uci.ics.amber.operator.ifStatement -import edu.uci.ics.amber.engine.common.executor.OperatorExecutor -import edu.uci.ics.amber.engine.common.model.State -import edu.uci.ics.amber.engine.common.model.tuple.{Tuple, TupleLike} -import edu.uci.ics.amber.engine.common.workflow.PortIdentity +import edu.uci.ics.amber.core.executor.OperatorExecutor +import edu.uci.ics.amber.core.marker.State +import edu.uci.ics.amber.core.tuple.{Tuple, TupleLike} +import edu.uci.ics.amber.workflow.PortIdentity class IfOpExec(conditionName: String) extends OperatorExecutor { private var outputPort: PortIdentity = PortIdentity(1) // by default, it should be the true port. From 2749a1d1312c75498ba86d8191937101335fdb11 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Tue, 17 Dec 2024 00:13:38 -0800 Subject: [PATCH 17/17] update --- .../uci/ics/amber/operator/LogicalOp.scala | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala index 1d2551b4f4b..6d37d51ef35 100644 --- a/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala +++ b/core/workflow-operator/src/main/scala/edu/uci/ics/amber/operator/LogicalOp.scala @@ -14,14 +14,22 @@ import edu.uci.ics.amber.operator.distinct.DistinctOpDesc import edu.uci.ics.amber.operator.dummy.DummyOpDesc import edu.uci.ics.amber.operator.filter.SpecializedFilterOpDesc import edu.uci.ics.amber.operator.hashJoin.HashJoinOpDesc -import edu.uci.ics.amber.operator.huggingFace.{HuggingFaceIrisLogisticRegressionOpDesc, HuggingFaceSentimentAnalysisOpDesc, HuggingFaceSpamSMSDetectionOpDesc, HuggingFaceTextSummarizationOpDesc} +import edu.uci.ics.amber.operator.huggingFace.{ + HuggingFaceIrisLogisticRegressionOpDesc, + HuggingFaceSentimentAnalysisOpDesc, + HuggingFaceSpamSMSDetectionOpDesc, + HuggingFaceTextSummarizationOpDesc +} import edu.uci.ics.amber.operator.ifStatement.IfOpDesc import edu.uci.ics.amber.operator.intersect.IntersectOpDesc import edu.uci.ics.amber.operator.intervalJoin.IntervalJoinOpDesc import edu.uci.ics.amber.operator.keywordSearch.KeywordSearchOpDesc import edu.uci.ics.amber.operator.limit.LimitOpDesc import edu.uci.ics.amber.operator.machineLearning.Scorer.MachineLearningScorerOpDesc -import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{SklearnAdvancedKNNClassifierTrainerOpDesc, SklearnAdvancedKNNRegressorTrainerOpDesc} +import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.KNNTrainer.{ + SklearnAdvancedKNNClassifierTrainerOpDesc, + SklearnAdvancedKNNRegressorTrainerOpDesc +} import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVCTrainer.SklearnAdvancedSVCTrainerOpDesc import edu.uci.ics.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.SklearnAdvancedSVRTrainerOpDesc import edu.uci.ics.amber.operator.metadata.{OPVersion, OperatorInfo, PropertyNameConstants} @@ -31,11 +39,42 @@ import edu.uci.ics.amber.operator.regex.RegexOpDesc import edu.uci.ics.amber.operator.reservoirsampling.ReservoirSamplingOpDesc import edu.uci.ics.amber.operator.sentiment.SentimentAnalysisOpDesc import edu.uci.ics.amber.operator.sink.managed.ProgressiveSinkOpDesc -import edu.uci.ics.amber.operator.sklearn.{SklearnAdaptiveBoostingOpDesc, SklearnBaggingOpDesc, SklearnBernoulliNaiveBayesOpDesc, SklearnComplementNaiveBayesOpDesc, SklearnDecisionTreeOpDesc, SklearnDummyClassifierOpDesc, SklearnExtraTreeOpDesc, SklearnExtraTreesOpDesc, SklearnGaussianNaiveBayesOpDesc, SklearnGradientBoostingOpDesc, SklearnKNNOpDesc, SklearnLinearRegressionOpDesc, SklearnLinearSVMOpDesc, SklearnLogisticRegressionCVOpDesc, SklearnLogisticRegressionOpDesc, SklearnMultiLayerPerceptronOpDesc, SklearnMultinomialNaiveBayesOpDesc, SklearnNearestCentroidOpDesc, SklearnPassiveAggressiveOpDesc, SklearnPerceptronOpDesc, SklearnPredictionOpDesc, SklearnProbabilityCalibrationOpDesc, SklearnRandomForestOpDesc, SklearnRidgeCVOpDesc, SklearnRidgeOpDesc, SklearnSDGOpDesc, SklearnSVMOpDesc} +import edu.uci.ics.amber.operator.sklearn.{ + SklearnAdaptiveBoostingOpDesc, + SklearnBaggingOpDesc, + SklearnBernoulliNaiveBayesOpDesc, + SklearnComplementNaiveBayesOpDesc, + SklearnDecisionTreeOpDesc, + SklearnDummyClassifierOpDesc, + SklearnExtraTreeOpDesc, + SklearnExtraTreesOpDesc, + SklearnGaussianNaiveBayesOpDesc, + SklearnGradientBoostingOpDesc, + SklearnKNNOpDesc, + SklearnLinearRegressionOpDesc, + SklearnLinearSVMOpDesc, + SklearnLogisticRegressionCVOpDesc, + SklearnLogisticRegressionOpDesc, + SklearnMultiLayerPerceptronOpDesc, + SklearnMultinomialNaiveBayesOpDesc, + SklearnNearestCentroidOpDesc, + SklearnPassiveAggressiveOpDesc, + SklearnPerceptronOpDesc, + SklearnPredictionOpDesc, + SklearnProbabilityCalibrationOpDesc, + SklearnRandomForestOpDesc, + SklearnRidgeCVOpDesc, + SklearnRidgeOpDesc, + SklearnSDGOpDesc, + SklearnSVMOpDesc +} import edu.uci.ics.amber.operator.sort.SortOpDesc import edu.uci.ics.amber.operator.sortPartitions.SortPartitionsOpDesc import edu.uci.ics.amber.operator.source.apis.reddit.RedditSearchSourceOpDesc -import edu.uci.ics.amber.operator.source.apis.twitter.v2.{TwitterFullArchiveSearchSourceOpDesc, TwitterSearchSourceOpDesc} +import edu.uci.ics.amber.operator.source.apis.twitter.v2.{ + TwitterFullArchiveSearchSourceOpDesc, + TwitterSearchSourceOpDesc +} import edu.uci.ics.amber.operator.source.fetcher.URLFetcherOpDesc import edu.uci.ics.amber.operator.source.scan.FileScanSourceOpDesc import edu.uci.ics.amber.operator.source.scan.csv.CSVScanSourceOpDesc