Merge pull request #234 from h2oai/develop

Release 1.1.3

ReleaseNotes.md

@@ -10,6 +10,14 @@ If you're an Open Source community member, you can contact H2O using one of the
 - Send an e-mail message directly to <a href="mailto:[email protected]">[email protected]</a>
 - Ask your question on the [H2O Community](https://community.h2o.ai/spaces/540/index.html) site (create an account if necessary)
 
+## v1.1.3 Changes
+
+- [STEAM-567] Updated Prediction Service UI to reflect DeepWater support
+- [STEAM-578] Added support for importing deploying DeepWater models
+- [STEAM-579] Added a default limit for CLI items to 10000
+- [STEAM-504] Modify existing permissions from UMS screen 
+- [STEAM-559] Prediction Service now allows binary input (images, audio, etc...)
+
 ## v1.1.2 Changes
 
 - [STEAM-574] Fixes missing protocol in Cluster listing addresses

cli2/cli.go

@@ -1274,7 +1274,7 @@ func findModels(c *context) *cobra.Command {
 	cmd.Flags().BoolVar(&regression, "regression", regression, "List regression models")
 
 	cmd.Flags().BoolVar(&ascending, "ascending", ascending, "No description available")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().StringVar(&namePart, "name-part", namePart, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	cmd.Flags().Int64Var(&projectId, "project-id", projectId, "No description available")
@@ -1662,7 +1662,7 @@ func getClusters(c *context) *cobra.Command {
 		return
 	})
 
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	return cmd
 }
@@ -1822,7 +1822,7 @@ func getDatasets(c *context) *cobra.Command {
 
 	cmd.Flags().Int64Var(&clusterId, "cluster-id", clusterId, "No description available")
 	cmd.Flags().Int64Var(&datasourceId, "datasource-id", datasourceId, "No description available")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	return cmd
 }
@@ -1913,7 +1913,7 @@ func getDatasources(c *context) *cobra.Command {
 		return
 	})
 
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	cmd.Flags().Int64Var(&projectId, "project-id", projectId, "No description available")
 	return cmd
@@ -2040,7 +2040,7 @@ func getHistory(c *context) *cobra.Command {
 
 	cmd.Flags().Int64Var(&entityId, "entity-id", entityId, "Integer ID for an entity in Steam.")
 	cmd.Flags().Int64Var(&entityTypeId, "entity-type-id", entityTypeId, "Integer ID for the type of entity.")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "The maximum returned objects.")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "The maximum returned objects.")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "An offset to start the search on.")
 	return cmd
 }
@@ -2183,7 +2183,7 @@ func getIdentities(c *context) *cobra.Command {
 
 	cmd.Flags().Int64Var(&entityId, "entity-id", entityId, "An entity ID.")
 	cmd.Flags().Int64Var(&entityType, "entity-type", entityType, "An entity type ID.")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "The maximum returned objects.")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "The maximum returned objects.")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "An offset to start the search on.")
 	cmd.Flags().Int64Var(&roleId, "role-id", roleId, "Integer ID of a role in Steam.")
 	cmd.Flags().Int64Var(&workgroupId, "workgroup-id", workgroupId, "Integer ID of a workgroup in Steam.")
@@ -2675,7 +2675,7 @@ func getModels(c *context) *cobra.Command {
 
 	cmd.Flags().Int64Var(&clusterId, "cluster-id", clusterId, "No description available")
 	cmd.Flags().StringVar(&frameKey, "frame-key", frameKey, "No description available")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	cmd.Flags().Int64Var(&projectId, "project-id", projectId, "No description available")
 	return cmd
@@ -2974,7 +2974,7 @@ func getProjects(c *context) *cobra.Command {
 		return
 	})
 
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	return cmd
 }
@@ -3116,7 +3116,7 @@ func getRoles(c *context) *cobra.Command {
 	cmd.Flags().BoolVar(&forIdentity, "for-identity", forIdentity, "List roles for an identity")
 
 	cmd.Flags().Int64Var(&identityId, "identity-id", identityId, "Integer ID of an identity in Steam.")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "The maximum returned objects.")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "The maximum returned objects.")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "An offset to start the search on.")
 	return cmd
 }
@@ -3282,7 +3282,7 @@ func getServices(c *context) *cobra.Command {
 	cmd.Flags().BoolVar(&forProject, "for-project", forProject, "List services for a project")
 	cmd.Flags().BoolVar(&forModel, "for-model", forModel, "List services for a model")
 
-	cmd.Flags().Int64Var(&limit, "limit", limit, "No description available")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "No description available")
 	cmd.Flags().Int64Var(&modelId, "model-id", modelId, "No description available")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "No description available")
 	cmd.Flags().Int64Var(&projectId, "project-id", projectId, "No description available")
@@ -3426,7 +3426,7 @@ func getWorkgroups(c *context) *cobra.Command {
 	cmd.Flags().BoolVar(&forIdentity, "for-identity", forIdentity, "List workgroups for an identity")
 
 	cmd.Flags().Int64Var(&identityId, "identity-id", identityId, "Integer ID of an identity in Steam.")
-	cmd.Flags().Int64Var(&limit, "limit", limit, "The maximum returned objects.")
+	cmd.Flags().Int64Var(&limit, "limit", 10000, "The maximum returned objects.")
 	cmd.Flags().Int64Var(&offset, "offset", offset, "An offset to start the search on.")
 	return cmd
 }

docs/conf.py

@@ -63,7 +63,7 @@
 # built documents.
 #
 # The short X.Y version.
-version = "Steam 1.1.2"
+version = "Steam 1.1.3"
 
 # This commented out code is how the version is displayed in H2O.
 #if os.path.exists("project_version"):

gui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "steam",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "",
   "main": "src/main.tsx",
   "dependencies": {

lib/fs/fs.go

@@ -37,6 +37,7 @@ import (
 	"time"
 
 	uuid "github.com/nu7hatch/gouuid"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -306,6 +307,10 @@ func GetGenModelPath(wd string, modelId int64) string {
 	return path.Join(GetModelPath(wd, modelId), "h2o-genmodel.jar")
 }
 
+func GetDeepwaterDepPath(wd string, modelId int64) string {
+	return path.Join(GetModelPath(wd, modelId), "deepwater-all.jar")
+}
+
 func GetAssetsPath(wd, asset string) string {
 	return path.Join(wd, AssetsDir, asset)
 }
@@ -493,16 +498,16 @@ func Download(p, u string, preserveFilename bool) (int64, string, error) {
 		return 0, "", fmt.Errorf("Download directory creation failed: %s: %v", p, err)
 	}
 
-	if _, err := os.Stat(p); !os.IsNotExist(err) {
-		return 0, "", fmt.Errorf("Download file creation failed: file already exists: %s: %v", p, err)
-	}
-
-	dst, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, FilePerm)
+	dst, err := os.Create(p)
 	if err != nil {
 		return 0, "", fmt.Errorf("Download file creation failed: %s: %v", p, err)
 	}
 	defer dst.Close()
 
+	if err := os.Chmod(p, FilePerm); err != nil {
+		return 0, "", errors.Wrap(err, "setting file permissions")
+	}
+
 	size, err := io.Copy(dst, res.Body)
 	if err != nil {
 		return 0, "", fmt.Errorf("Download file copy failed: %s to %s: %v", u, p, err)

master/download.go

@@ -197,8 +197,9 @@ func (s *DownloadHandler) serveModel(w http.ResponseWriter, r *http.Request, pz
 				projectId,
 				modelId,
 				model.LogicalName,
-				compiler.ArtifactWar,
 				"pojo",
+				model.Algorithm,
+				compiler.ArtifactWar,
 				"",
 			)
 			if err != nil {
@@ -214,8 +215,9 @@ func (s *DownloadHandler) serveModel(w http.ResponseWriter, r *http.Request, pz
 				projectId,
 				modelId,
 				model.LogicalName,
-				compiler.ArtifactWar,
 				"mojo",
+				model.Algorithm,
+				compiler.ArtifactWar,
 				"",
 			)
 			if err != nil {
@@ -235,8 +237,9 @@ func (s *DownloadHandler) serveModel(w http.ResponseWriter, r *http.Request, pz
 				projectId,
 				modelId,
 				model.LogicalName,
-				compiler.ArtifactPythonWar,
 				"pojo",
+				model.Algorithm,
+				compiler.ArtifactPythonWar,
 				packageName,
 			)
 			if err != nil {
@@ -257,8 +260,9 @@ func (s *DownloadHandler) serveModel(w http.ResponseWriter, r *http.Request, pz
 				projectId,
 				modelId,
 				model.LogicalName,
-				compiler.ArtifactPythonWar,
 				"mojo",
+				model.Algorithm,
+				compiler.ArtifactPythonWar,
 				packageName,
 			)
 			if err != nil {
@@ -273,8 +277,9 @@ func (s *DownloadHandler) serveModel(w http.ResponseWriter, r *http.Request, pz
 				projectId,
 				modelId,
 				model.LogicalName,
-				compiler.ArtifactJar,
 				"pojo",
+				model.Algorithm,
+				compiler.ArtifactJar,
 				"",
 			)
 			if err != nil {

master/web/service.go

@@ -1252,7 +1252,7 @@ func (s *Service) createMetricsTable(pz az.Principal, modelId int64, metrics *bi
 
 func (s *Service) CheckMojo(pz az.Principal, algo string) (bool, error) {
 	switch algo {
-	case "Gradient Boosting Method", "Distributed Random Forest":
+	case "Gradient Boosting Method", "Distributed Random Forest", "Deep Water":
 		return true, nil
 	}
 	return false, nil
@@ -1320,6 +1320,13 @@ func (s *Service) ImportModelMojo(pz az.Principal, modelId int64) error {
 		return errors.Wrap(err, "failed to export java dependency")
 	}
 
+	// MOJO only check
+	if m.Algorithm == "Deep Water" {
+		if _, err := h2o.ExportDeepWaterAll(modelPath); err != nil {
+			return errors.Wrap(err, "exporting Deep Water dependency")
+		}
+	}
+
 	if !m.LogicalName.Valid {
 		s.ds.UpdateModelLocation(pz, modelId, strconv.FormatInt(modelId, 10), fs.GetBasenameWithoutExt(mojoPath))
 	}
@@ -1510,8 +1517,9 @@ func (s *Service) StartService(pz az.Principal, modelId int64, name, packageName
 		model.ProjectId,
 		model.Id,
 		model.LogicalName.String,
-		artifact,
 		model.ModelObjectType.String,
+		model.Algorithm,
+		artifact,
 		packageName,
 	)
 	if err != nil {

prediction-service-builder/examples/deepwater/example.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+rm -f example.war
+
+curl -X POST \
+--form [email protected] \
+--form [email protected] \
+--form [email protected] \
+localhost:55000/makewar > example.war
+
+if [ -s example.war ]
+then
+  echo "Created example.war"
+  echo "Run with run-example.sh"
+else
+  echo "Failed to build example.war"
+  exit 1
+fi

prediction-service-builder/examples/deepwater/run-debug.sh

@@ -0,0 +1 @@
+java -Dorg.slf4j.simpleLogger.defaultLogLevel=DEBUG -jar ../jetty-runner-8.1.14.v20131031.jar --port 55001 example.war 

prediction-service-builder/examples/deepwater/run-example.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Starting prediction service on port 55001"
+echo ""
+
+java -jar ../jetty-runner-8.1.14.v20131031.jar --port 55001 example.war 
+

prediction-service-builder/examples/deepwater/test-example.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+
+curl -X POST \
+--form [email protected] \
+--form abc=def \
+--form data='{Dest: SFO, Orig: JFK}' \
+http://localhost:55001/predictbinary
+
+echo "test1 bird"
+
+curl -X POST \
+--form [email protected] \
+http://localhost:55001/predictbinary
+
+echo "test2 dog"
+
+curl -X POST \
+--form [email protected] \
+http://localhost:55001/predictbinary
+
+echo "url of dog"
+
+#curl -X POST \
+#--form C1="https://c1.staticflickr.com/1/225/515776742_bce2e6dbea_d.jpg" \
+#http://localhost:55001/predictbinary

prediction-service-builder/src/main/java/ai/h2o/servicebuilder/MakeWarServlet.java

@@ -92,6 +92,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
       String pojofile = null;
       String jarfile = null;
       String prejarfile = null;
+      String deepwaterjarfile = null;
       String rawfile = null;
       String predictorClassName = null;
       String transformerClassName = null;
@@ -112,6 +113,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
             jarfile = "WEB-INF" + File.separator + "lib" + File.separator + filename;
             FileUtils.copyInputStreamToFile(i.getInputStream(), new File(libDir, filename));
           }
+          if (field.equals("deepwater")) {
+            deepwaterjarfile = "WEB-INF" + File.separator + "lib" + File.separator + filename;
+            FileUtils.copyInputStreamToFile(i.getInputStream(), new File(libDir, filename));
+          }
           if (field.equals("prejar")) {
             prejarfile = "WEB-INF" + File.separator + "lib" + File.separator + filename;
             FileUtils.copyInputStreamToFile(i.getInputStream(), new File(libDir, filename));
@@ -130,8 +135,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
           }
         }
       }
-      logger.debug("genmodeljar {}  pojo {}  raw {}", jarfile, pojofile, rawfile);
-//      if (pojofile == null || jarfile == null)
+      logger.debug("genmodeljar {}  deepwaterjar {}  pojo {}  raw {}", jarfile, deepwaterjarfile, pojofile, rawfile);
       if ((pojofile == null || jarfile == null) && (rawfile == null || jarfile == null))
         throw new Exception("need either pojo and genmodel jar, or raw file and genmodel jar ");
 
@@ -141,8 +145,11 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
 
       if (pojofile != null) {
         // Compile the pojo
+        String jarfiles = jarfile;
+        if (deepwaterjarfile != null)
+          jarfiles += ":" + deepwaterjarfile;
         runCmd(tmpDir, Arrays.asList("javac", "-target", JAVA_TARGET_VERSION, "-source", JAVA_TARGET_VERSION,
-            "-J-Xmx" + MEMORY_FOR_JAVA_PROCESSES, "-cp", jarfile, "-d", outDir.getPath(), pojofile),
+            "-J-Xmx" + MEMORY_FOR_JAVA_PROCESSES, "-cp", jarfiles, "-d", outDir.getPath(), pojofile),
             "Compilation of pojo failed");
         logger.info("compiled pojo {}", pojofile);
       }
@@ -186,6 +193,7 @@ else if (!rawfiles.isEmpty()) {
       InstantiateJavaTemplateFile(tmpDir, modelCode, predictorClassName, replaceTransform, srcPath + "ServletUtil-TEMPLATE.java", "ServletUtil.java");
 
       copyExtraFile(servletPath, srcPath, tmpDir, "PredictServlet.java", "PredictServlet.java");
+      copyExtraFile(servletPath, srcPath, tmpDir, "PredictBinaryServlet.java", "PredictBinaryServlet.java");
       copyExtraFile(servletPath, srcPath, tmpDir, "InfoServlet.java", "InfoServlet.java");
       copyExtraFile(servletPath, srcPath, tmpDir, "StatsServlet.java", "StatsServlet.java");
       copyExtraFile(servletPath, srcPath, tmpDir, "PingServlet.java", "PingServlet.java");
@@ -195,7 +203,8 @@ else if (!rawfiles.isEmpty()) {
       // compile extra
       List<String> cmd = Arrays.asList("javac", "-target", JAVA_TARGET_VERSION, "-source", JAVA_TARGET_VERSION, "-J-Xmx" + MEMORY_FOR_JAVA_PROCESSES,
           "-cp", "WEB-INF/lib/*:WEB-INF/classes:extra/WEB-INF/lib/*", "-d", outDir.getPath(),
-          "PredictServlet.java", "InfoServlet.java", "StatsServlet.java", "ServletUtil.java", "PingServlet.java", "Transform.java", "Logging.java");
+          "PredictServlet.java", "PredictBinaryServlet.java", "InfoServlet.java", "StatsServlet.java", "ServletUtil.java",
+          "PingServlet.java", "Transform.java", "Logging.java");
       runCmd(tmpDir, cmd, "Compilation of extra failed");
 
       // create the war jar file