Add plugins/vars for multi-cluster GCS consistency

The standalone_nfs_cache_env.sh can be used to turn up a dedicated NFS server
for list-consistency purposes; setting GCS_CACHE_MASTER_HOSTNAME in
the variable overrides for subsequent clusters will then point at the
specified host for the NFS cache rather than assuming the cluster's
master is in charge of the NFS cache.

CHANGES.txt

@@ -1,3 +1,11 @@
+1.3.2 - 2015-xx-xx
+
+  1. Added support for setting up a standalone NFS cache server for GCS
+     consistency using standalone_nfs_cache_env.sh, along with configurable
+     GCS_CACHE_MASTER_HOSTNAME to point subsequent clusters at the shared
+     NFS cache server. See standalone_nfs_cache_env.sh for usage.
+
+
 1.3.1 - 2015-07-09
 
   1. Added plugin for deploying MapR under platforms/mapr/mapr_env.sh; see

bdutil_env.sh

@@ -157,6 +157,13 @@ DEFAULT_FS='gs'
 # list-after-write consistency.
 ENABLE_NFS_GCS_FILE_CACHE=true
 
+# If set, uses the provided hostname for the NFS-based GCS consistency cache
+# rather than assuming the master on the cluster is always the cache server.
+# This allows a standalone cache (optionally createed with the bdutil-provided
+# standalone_nfs_cache_env.sh) to be used cross-cluster. If unset, defaults
+# to using the master of the cluster as the cache server.
+GCS_CACHE_MASTER_HOSTNAME=''
+
 # User to create which owns the directories used by the NFS GCS file cache
 # and potentially other gcs-connector-related tasks.
 GCS_ADMIN='gcsadmin'
@@ -359,6 +366,12 @@ function evaluate_late_variable_bindings() {
   # GCS directory for deployment-related temporary files.
   local staging_dir_base="gs://${CONFIGBUCKET}/bdutil-staging"
   BDUTIL_GCS_STAGING_DIR="${staging_dir_base}/${MASTER_HOSTNAME}"
+
+  # Default NFS cache host is the master node, but it can be overriden to point
+  # at an NFS server off-cluster.
+  if [[ -z "${GCS_CACHE_MASTER_HOSTNAME}" ]]; then
+    GCS_CACHE_MASTER_HOSTNAME="${MASTER_HOSTNAME}"
+  fi
 }
 
 # Helper to allow env_file dependency. Relative paths are resolved relative to

libexec/setup_client_nfs.sh

@@ -30,7 +30,7 @@ if (( ${INSTALL_GCS_CONNECTOR} )) && \
   fi
 
   MOUNT_STRING="/${NFS_MOUNT_POINT} -fstype=nfs,defaults,rw,hard,intr"
-  MOUNT_STRING="${MOUNT_STRING} ${MASTER_HOSTNAME}:${NFS_EXPORT_POINT}"
+  MOUNT_STRING="${MOUNT_STRING} ${GCS_CACHE_MASTER_HOSTNAME}:${NFS_EXPORT_POINT}"
   echo "${MOUNT_STRING}" > /etc/auto.hadoop_gcs_metadata_cache
 
   if [[ -f /usr/lib/systemd/system/autofs.service ]] \

libexec/setup_master_nfs.sh

@@ -15,7 +15,8 @@
 # Installs NFS packages and exports on the master.
 
 if (( ${INSTALL_GCS_CONNECTOR} )) && \
-   (( ${ENABLE_NFS_GCS_FILE_CACHE} )) ; then
+   (( ${ENABLE_NFS_GCS_FILE_CACHE} )) && \
+   [[ "$(hostname -s)" == "${GCS_CACHE_MASTER_HOSTNAME}" ]] ; then
   # Set up the GCS_ADMIN user.
   setup_gcs_admin
 

single_node_env.sh

@@ -27,6 +27,11 @@ INSTALL_JDK_DEVEL=true
 copy_func evaluate_late_variable_bindings old_evaluate_late_variable_bindings
 
 function evaluate_late_variable_bindings() {
+  # Stash away the old value here so we can differentiate between whether the
+  # user overrides set it or we just resolved it in the base implementation
+  # of evaluate_late_variable_bindings.
+  local old_nfs_master_hostname="${GCS_CACHE_MASTER_HOSTNAME}"
+
   old_evaluate_late_variable_bindings
 
   # In the case of the single-node cluster, we'll just use the whole PREFIX
@@ -46,6 +51,12 @@ function evaluate_late_variable_bindings() {
   local staging_dir_base="gs://${CONFIGBUCKET}/bdutil-staging"
   BDUTIL_GCS_STAGING_DIR="${staging_dir_base}/${MASTER_HOSTNAME}"
 
+  # Default NFS cache host is the master node, but it can be overriden to point
+  # at an NFS server off-cluster.
+  if [[ -z "${old_nfs_master_hostname}" ]]; then
+    GCS_CACHE_MASTER_HOSTNAME="${MASTER_HOSTNAME}"
+  fi
+
   # Since $WORKERS and $MASTER_HOSTNAME both refer to the same single-node
   # VM, we must override COMMAND_STEPS to prevent duplicating steps. We also
   # omit deploy-ssh-worker-setup because there is no need to copy SSH keys to

standalone_nfs_cache_env.sh

@@ -0,0 +1,68 @@
+# Copyright 2015 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Handy wrapper around single_node_env.sh to turn up just a single server
+# capable of acting as the NFS-based GCS consistency cache for multiple
+# other clusters.
+#
+# Usage:
+#   ./bdutil -P my-nfs-server -p <project> -z <zone> -b <bucket> generate_config my-nfs-server_env.sh
+#   ./bdutil -e my-nfs-server_env.sh deploy
+#
+#   ./bdutil -P cluster1 -p <project> -z <zone> -b <bucket> generate_config cluster1_env.sh
+#   echo GCS_CACHE_MASTER_HOSTNAME=my-nfs-server >> cluster1_env.sh
+#   ./bdutil -e cluster1_env.sh deploy
+#
+#   ./bdutil -P cluster2 -p <project> -z <zone> -b <bucket> generate_config cluster2_env.sh
+#   echo GCS_CACHE_MASTER_HOSTNAME=my-nfs-server >> cluster2_env.sh
+#   ./bdutil -e cluster2_env.sh deploy
+#
+#  ./bdutil -e cluster2_env.sh delete
+#  ./bdutil -e cluster1_env.sh delete
+#  ./bdutil -e my-nfs-server_env.sh delete
+
+# Start with single_node_env.sh to get all the MASTER_HOSTNAME, etc.,
+# resolution.
+import_env single_node_env.sh
+
+# This server would be somewhat pointless without the GCS connector and the
+# NFS cache enabled.
+INSTALL_GCS_CONNECTOR=true
+DEFAULT_FS='gs'
+ENABLE_NFS_GCS_FILE_CACHE=true
+
+# We'll set up Hadoop as normal since it'll be handy to have "hadoop fs -ls"
+# on the cache server, but we just won't configure the hadoop daemons to start
+# on boot, and won't start them explicitly during deployment. That means
+# no jobracker or resourcemanager or namenode, but we should still be able to
+# use "hadoop fs" against GCS just fine.
+COMMAND_GROUPS+=(
+  "deploy-standalone-nfs-cache:
+     libexec/install_java.sh
+     libexec/mount_disks.sh
+     libexec/setup_hadoop_user.sh
+     libexec/install_hadoop.sh
+     libexec/install_bdconfig.sh
+     libexec/configure_hadoop.sh
+     libexec/install_and_configure_gcs_connector.sh
+     libexec/configure_hdfs.sh
+     libexec/set_default_fs.sh
+     libexec/setup_master_nfs.sh
+  "
+)
+
+COMMAND_STEPS=(
+  "deploy-standalone-nfs-cache,*"
+  "deploy-client-nfs-setup,*"
+)