[Improve] Limit the number of running build projects #3687 (#3695)

* [Improve] Limit the number of running build projects #3687 --------- Co-authored-by: benjobs <[email protected]>
apache · Apr 30, 2024 · c3c468c · c3c468c
1 parent eae3265
commit c3c468c
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 26 deletions.
diff --git a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
@@ -63,6 +63,11 @@
             <outputDirectory>lib</outputDirectory>
             <fileMode>0755</fileMode>
         </fileSet>
+        <fileSet>
+            <directory>${project.build.directory}/../src/main/assembly/conf</directory>
+            <outputDirectory>conf</outputDirectory>
+            <fileMode>0755</fileMode>
+        </fileSet>
         <fileSet>
             <directory>${project.build.directory}/../src/main/assembly/logs</directory>
             <outputDirectory>logs</outputDirectory>
@@ -95,7 +100,6 @@
             <lineEnding>unix</lineEnding>
             <fileMode>0755</fileMode>
             <includes>
-                <include>config.yaml</include>
                 <include>logback-spring.xml</include>
             </includes>
         </fileSet>

diff --git a/...le-service/src/main/resources/config.yaml → ...ervice/src/main/assembly/conf/config.yaml b/...le-service/src/main/resources/config.yaml → ...ervice/src/main/assembly/conf/config.yaml
@@ -61,6 +61,9 @@ streampark:
     http-auth: 'simple'  # default simple, or kerberos
   # flink on yarn or spark on yarn, HADOOP_USER_NAME
   hadoop-user-name: hdfs
+  project:
+    # Number of projects allowed to be running at the same time , If there is no limit, -1 can be configured
+    max-build: 16
 
 # flink on yarn or spark on yarn, when the hadoop cluster enable kerberos authentication, it is necessary to set Kerberos authentication parameters.
 security:

diff --git a/...ole-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java b/...ole-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
@@ -17,6 +17,7 @@
 
 package org.apache.streampark.console.base.config;
 
+import org.apache.streampark.common.conf.ConfigConst;
 import org.apache.streampark.common.util.PropertiesUtils;
 import org.apache.streampark.common.util.SystemPropertyUtils;
 import org.apache.streampark.console.base.util.WebUtils;
@@ -28,7 +29,6 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.io.InputStream;
 import java.util.Map;
 import java.util.Properties;
 
@@ -46,10 +46,10 @@ public static Properties get() {
       SystemPropertyUtils.set("spring.config.location", oldConfig.getAbsolutePath());
       return new Properties();
     } else {
-      // 1) get spring config
-      Properties springConfig = getSpringConfig();
-      // 2) get user config
+      // 1) get user config
       Properties userConfig = getUserConfig();
+      // 2) get spring config
+      Properties springConfig = getSpringConfig();
       // 3) merge config
       mergeConfig(userConfig, springConfig);
       // 4) datasource
@@ -122,32 +122,24 @@ private static void mergeConfig(Properties userConfig, Properties springConfig)
         });
   }
 
-  private static boolean useOldConfig() {
-    String appHome = WebUtils.getAppHome();
-    if (appHome == null) {
-      return false;
-    }
-    File file = new File(appHome + "/conf/application.yml");
-    return file.exists();
-  }
-
   private static Properties getUserConfig() {
     String appHome = WebUtils.getAppHome();
+    if (StringUtils.isBlank(appHome)) {
+      throw new ExceptionInInitializerError(
+          String.format(
+              "[StreamPark] The system initialization check failed. If started local for development and debugging,"
+                  + " please ensure the -D%s parameter is clearly specified,"
+                  + " more detail: https://streampark.apache.org/docs/user-guide/deployment",
+              ConfigConst.KEY_APP_HOME()));
+    }
     Properties properties = new Properties();
-    if (appHome != null) {
-      File file = new File(appHome + "/conf/config.yaml");
-      if (file.exists() && file.isFile()) {
-        Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
-        properties.putAll(config);
-        return properties;
-      }
-      throw new ExceptionInInitializerError(file.getAbsolutePath() + " not found, please check.");
-    } else {
-      InputStream inputStream =
-          SpringProperties.class.getClassLoader().getResourceAsStream("config.yaml");
-      Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(inputStream);
+    File file = new File(appHome, "conf/config.yaml");
+    if (file.exists() && file.isFile()) {
+      Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
       properties.putAll(config);
       return properties;
+    } else {
+      throw new ExceptionInInitializerError(file.getAbsolutePath() + " not found, please check.");
     }
   }
 

diff --git a/...onsole-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java b/...onsole-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java
@@ -38,4 +38,6 @@ public interface ProjectMapper extends BaseMapper<Project> {
   Boolean existsByTeamId(@Param("teamId") Long teamId);
 
   List<Project> selectByTeamId(@Param("teamId") Long teamId);
+
+  Long getBuildingCount();
 }
diff --git a/...ice/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/...ice/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -50,6 +50,7 @@
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
 import org.springframework.transaction.annotation.Transactional;
@@ -83,6 +84,9 @@ public class ProjectServiceImpl extends ServiceImpl<ProjectMapper, Project>
 
   @Autowired private FlinkAppHttpWatcher flinkAppHttpWatcher;
 
+  @Value("${streampark.project.max-build:6}")
+  public Long maxProjectBuildNum;
+
   private static final int CPU_NUM = Math.max(4, Runtime.getRuntime().availableProcessors() * 2);
 
   private final ExecutorService projectBuildExecutor =
@@ -193,6 +197,14 @@ public List<Project> findByTeamId(Long teamId) {
 
   @Override
   public void build(Long id) throws Exception {
+    Long currentBuildCount = this.baseMapper.getBuildingCount();
+
+    ApiAlertException.throwIfTrue(
+        maxProjectBuildNum > -1 && currentBuildCount > maxProjectBuildNum,
+        String.format(
+            "The number of running Build projects exceeds the maximum number: %d of max-build-num",
+            maxProjectBuildNum));
+
     Project project = getById(id);
     this.baseMapper.updateBuildState(project.getId(), BuildState.BUILDING.get());
     String logPath = getBuildLogPath(id);

diff --git a/...mpark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml b/...mpark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
@@ -85,4 +85,8 @@
         </where>
     </select>
 
+    <select id="getBuildingCount" resultType="java.lang.Long">
+        select count(1) from t_flink_project where build_state = 0
+    </select>
+
 </mapper>