diff --git a/dss-apps/dss-apps-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSAppsServerApplication.scala b/dss-apps/dss-apps-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSAppsServerApplication.scala index 32bc6d94a..18588139b 100644 --- a/dss-apps/dss-apps-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSAppsServerApplication.scala +++ b/dss-apps/dss-apps-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSAppsServerApplication.scala @@ -1,40 +1,55 @@ - /* - * - * * Copyright 2019 WeBank - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ +/* +* +* * Copyright 2019 WeBank +* * +* * Licensed under the Apache License, Version 2.0 (the "License"); +* * you may not use this file except in compliance with the License. +* * You may obtain a copy of the License at +* * +* * http://www.apache.org/licenses/LICENSE-2.0 +* * +* * Unless required by applicable law or agreed to in writing, software +* * distributed under the License is distributed on an "AS IS" BASIS, +* * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* * See the License for the specific language governing permissions and +* * limitations under the License. +* +*/ package com.webank.wedatasphere.dss.apps import com.webank.wedatasphere.dss.common.utils.DSSMainHelper import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.conf.DSSConfiguration import org.apache.linkis.common.utils.{Logging, Utils} +import java.io.File + object DSSAppsServerApplication extends Logging { val userName: String = System.getProperty("user.name") val hostName: String = Utils.getComputerName + val legacyServerConfigs: Array[String] = Array("dss-apiservice-server.properties", "dss-datapipe-server.properties", + "dss-guide-server.properties", "dss-scriptis-server.properties") + def main(args: Array[String]): Unit = { - val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + val serviceName = System.getProperty("serviceName") //ProjectConf.SERVICE_NAME.getValue DSSMainHelper.formatPropertyFiles(serviceName) + // 若dss-apps-server配置不存在,则加载先前配置 + val serverConfFileURL = getClass.getClassLoader.getResource(serviceName + ".properties") + if (serverConfFileURL == null || !new File(serverConfFileURL.getPath).exists) { + logger.info(s"$serviceName.properties is not exists, now try to load legacy configs.") + DSSConfiguration.addLegacyConfiguration(legacyServerConfigs) + // server.port使用dss-scriptis-server.properties设置的值 + DSSConfiguration.setSpringApplicationName("dss-apps-server") + } val allArgs = args ++ DSSMainHelper.getExtraSpringOptions System.setProperty("hostName", hostName) System.setProperty("userName", userName) info(s"Ready to start $serviceName with args: ${allArgs.toList}.") println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) } } \ No newline at end of file diff --git a/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala index f1a788709..b446e7ea1 100644 --- a/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala +++ b/dss-commons/dss-common/src/main/scala/org/apache/linkis/common/conf/DSSConfiguration.scala @@ -1,9 +1,77 @@ package org.apache.linkis.common.conf +import org.apache.commons.io.IOUtils +import org.apache.linkis.common.utils.{Logging, Utils} + +import java.io.{File, FileInputStream, IOException, InputStream} +import java.util import java.util.Properties +import scala.collection.JavaConverters._ + +object DSSConfiguration extends Logging { -object DSSConfiguration { + private val MAPPER_LOCATIONS = "wds.linkis.server.mybatis.mapperLocations" + private val TYPE_ALIASES_PACKAGE = "wds.linkis.server.mybatis.typeAliasesPackage" + private val BASE_PACKAGE = "wds.linkis.server.mybatis.BasePackage" + private val RESTFUL_SCAN_PACKAGES = "wds.linkis.server.restful.scan.packages" def getAllProperties: Properties = BDPConfiguration.properties + def addLegacyConfiguration(serverConfs: Array[String]): Unit = { + val config = new Properties + val mapperLocationList = new util.ArrayList[String](); + val typeAliasesPackageList = new util.ArrayList[String](); + val basePackageList = new util.ArrayList[String](); + val restfulScanPackagesList = new util.ArrayList[String](); + + serverConfs.foreach { serverConf => + val serverConfFileURL = getClass.getClassLoader.getResource(serverConf) + if (serverConfFileURL != null && new File(serverConfFileURL.getPath).exists) { + logger.info( + s"*********************** Notice: The DSS serverConf file is $serverConf ! ******************" + ) + initConfig(config, serverConfFileURL.getPath) + } else { + logger.warn( + s"**************** Notice: The DSS serverConf file $serverConf does not exist! *******************" + ) + } + } + config.asScala.toMap.foreach { case (k, v) => + k match { + case MAPPER_LOCATIONS => mapperLocationList.add(v) + case TYPE_ALIASES_PACKAGE => typeAliasesPackageList.add(v) + case BASE_PACKAGE => basePackageList.add(v) + case RESTFUL_SCAN_PACKAGES => restfulScanPackagesList.add(v) + case _ => BDPConfiguration.set(k, v) + } + } + BDPConfiguration.set(MAPPER_LOCATIONS, String.join(",", mapperLocationList)) + BDPConfiguration.set(TYPE_ALIASES_PACKAGE, String.join(",", typeAliasesPackageList)) + BDPConfiguration.set(BASE_PACKAGE, String.join(",", basePackageList)) + BDPConfiguration.set(RESTFUL_SCAN_PACKAGES, String.join(",", restfulScanPackagesList)) + } + + private def initConfig(config: Properties, filePath: String): Unit = { + var inputStream: InputStream = null + Utils.tryFinally { + Utils.tryCatch { + inputStream = new FileInputStream(filePath) + config.load(inputStream) + } { case e: IOException => + logger.error("Can't load " + filePath, e) + } + } { + IOUtils.closeQuietly(inputStream) + } + } + + def setConfig(key: String, value: String): Unit = { + BDPConfiguration.set(key, value) + } + + def setSpringApplicationName(name: String): Unit = { + BDPConfiguration.set("spring.spring.application.name", name) + } + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala index c4e467ec4..25add1c6e 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala @@ -25,6 +25,7 @@ import com.webank.wedatasphere.dss.orchestrator.publish.entity.OrchestratorExpor import com.webank.wedatasphere.dss.orchestrator.publish.{ExportDSSOrchestratorPlugin, ImportDSSOrchestratorPlugin} import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} import org.apache.linkis.rpc.{Receiver, Sender} +import org.slf4j.{Logger, LoggerFactory} import java.util import scala.concurrent.duration.Duration @@ -32,6 +33,8 @@ import scala.concurrent.duration.Duration class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestratorPluginService: OrchestratorPluginService, orchestratorContext: DSSOrchestratorContext) extends Receiver { + private val LOGGER = LoggerFactory.getLogger(classOf[DSSOrchestratorReceiver]) + override def receive(message: Any, sender: Sender): Unit = {} override def receiveAndReply(message: Any, sender: Sender): Any = message match { @@ -69,6 +72,7 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr case requestConversionOrchestration: RequestFrameworkConvertOrchestration => //发布调度,请注意 + LOGGER.info("received requestConversionOrchestration, the class is: {}", requestConversionOrchestration) orchestratorPluginService.convertOrchestration(requestConversionOrchestration) case requestConversionOrchestrationStatus: RequestFrameworkConvertOrchestrationStatus => orchestratorPluginService.getConvertOrchestrationStatus(requestConversionOrchestrationStatus.getId) diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/DSSServerApplication.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/DSSServerApplication.scala new file mode 100644 index 000000000..2bb9f8750 --- /dev/null +++ b/dss-server/src/main/scala/com/webank/wedatasphere/dss/DSSServerApplication.scala @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.conf.DSSConfiguration +import org.apache.linkis.common.utils.{Logging, Utils} + +import java.io.File + +object DSSServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + val legacyServerConfigs: Array[String] = Array("dss-framework-project-server.properties", "dss-framework-orchestrator-server.properties", + "dss-workflow-server.properties", "dss-flow-execution-server.properties") + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName") //ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + // 若dss-server配置不存在,则加载先前配置 + val serverConfFileURL = getClass.getClassLoader.getResource(serviceName + ".properties") + if (serverConfFileURL == null || !new File(serverConfFileURL.getPath).exists) { + logger.info(s"$serviceName.properties is not exists, now try to load legacy configs.") + DSSConfiguration.addLegacyConfiguration(legacyServerConfigs) + DSSConfiguration.setSpringApplicationName("dss-server-dev") + } + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSServerApplication.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSServerApplication.scala deleted file mode 100644 index 29acd525e..000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/apps/DSSServerApplication.scala +++ /dev/null @@ -1,40 +0,0 @@ - /* - * - * * Copyright 2019 WeBank - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.apps - -import com.webank.wedatasphere.dss.common.utils.DSSMainHelper -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.utils.{Logging, Utils} - -object DSSServerApplication extends Logging { - - val userName: String = System.getProperty("user.name") - val hostName: String = Utils.getComputerName - - def main(args: Array[String]): Unit = { - val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue - DSSMainHelper.formatPropertyFiles(serviceName) - val allArgs = args ++ DSSMainHelper.getExtraSpringOptions - System.setProperty("hostName", hostName) - System.setProperty("userName", userName) - info(s"Ready to start $serviceName with args: ${allArgs.toList}.") - println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") - DataWorkCloudApplication.main(allArgs) - } -} \ No newline at end of file diff --git a/sbin/common.sh b/sbin/common.sh index 567e07cde..5c3727507 100644 --- a/sbin/common.sh +++ b/sbin/common.sh @@ -161,15 +161,13 @@ status() } function setServerName(){ - if [[ $PROJECT_NAME == *"project"* ]]; then - SERVER_NAME=dss-framework-project-server - elif [[ $PROJECT_NAME == *"orchestrator"* ]]; then - SERVER_NAME=dss-framework-orchestrator-server + elif [[ $PROJECT_NAME == *"dss-server"* ]]; then + SERVER_NAME=dss-server elif [[ $PROJECT_NAME == *"apps"* ]]; then SERVER_NAME=dss-apps-server else - echo "please input: sh dss-daemon.sh [start,restart,stop] [server name]; for example : sh dss-daemon.sh restart project " - echo "server name : project、orchestrator、apps" + echo "please input: sh dss-daemon.sh [start,restart,stop] [server name]; for example : sh dss-daemon.sh restart dss-server " + echo "server name : dss-server、apps-server" exit 1 fi } diff --git a/sbin/dss-start-all.sh b/sbin/dss-start-all.sh index a6d4321e5..309dae8ce 100644 --- a/sbin/dss-start-all.sh +++ b/sbin/dss-start-all.sh @@ -80,55 +80,26 @@ echo "<-------------------------------->" sleep 3 } - function startDssProject(){ - SERVER_NAME=dss-framework-project-server - SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP - startApp - sleep 5 - -# echo "------------------------Start to check whether the project service is registered to eureka successfully-----------------------------" -# #project服务启动并注册到eureka后再启动其他服务 -# i=1 -# while [[ -z $result ]] && [[ $i -le 24 ]] -# do -# sleep 5 -# if [ -z $EUREKA_USERNAME ] || [ -z $EUREKA_PASSWORD ];then -# response=`curl http://${EUREKA_INSTALL_IP}:${EUREKA_PORT}/eureka/apps/DSS-FRAMEWORK-PROJECT-SERVER` -# else -# response=`curl http://${EUREKA_USENAME}:${EUREKA_PASSWORD}@${EUREKA_INSTALL_IP}:${EUREKA_PORT}/eureka/apps/DSS-FRAMEWORK-PROJECT-SERVER` -# fi -# let i++ -# result=$(echo $response |grep 'DSS-FRAMEWORK-PROJECT-SERVER') -# done -# if [[ $i -eq 25 ]]; then -# echo "the project server start failed in two minutes,please check the log to find more error details." -# exit -# fi -# echo "------------------------the project service is registered to eureka successfully------------------------------------------------" - SERVER_NAME=dss-apps-server SERVER_IP=$DSS_APPS_SERVER_INSTALL_IP startApp - SERVER_NAME=dss-framework-orchestrator-server - SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + SERVER_NAME=dss-server + SERVER_IP=$DSS_SERVER_INSTALL_IP startApp } function checkDssService(){ - SERVER_NAME=dss-framework-project-server - SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + SERVER_NAME=dss-apps-server + SERVER_IP=$DSS_APPS_SERVER_INSTALL_IP checkServer - SERVER_NAME=dss-framework-orchestrator-server - SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + SERVER_NAME=dss-server + SERVER_IP=$DSS_SERVER_INSTALL_IP checkServer - SERVER_NAME=dss-data-api-server - SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP - checkServer } diff --git a/sbin/dss-stop-all.sh b/sbin/dss-stop-all.sh index 28b1d82ed..b6f0b36d9 100644 --- a/sbin/dss-stop-all.sh +++ b/sbin/dss-stop-all.sh @@ -51,12 +51,8 @@ echo "<-------------------------------->" } function stopDssProject(){ - SERVER_NAME=dss-framework-project-server - SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP - stopApp - - SERVER_NAME=dss-framework-orchestrator-server - SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + SERVER_NAME=dss-server + SERVER_IP=$DSS_SERVER_INSTALL_IP stopApp SERVER_NAME=dss-apps-server diff --git a/sbin/ext/dss-server b/sbin/ext/dss-server new file mode 100644 index 000000000..fc33f2bb5 --- /dev/null +++ b/sbin/ext/dss-server @@ -0,0 +1,86 @@ +#!/bin/bash +# +# description: dss-server start cmd +# + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="6G" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.DSSServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1