From c63148de14ea8c5ffaba743a1fe748f854b7f8f2 Mon Sep 17 00:00:00 2001 From: Dario Berzano Date: Tue, 6 May 2014 14:38:24 +0200 Subject: [PATCH 1/7] Improved way of retrieving extra user-data --- hepix/commands/amiconfig.sh | 76 ++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/hepix/commands/amiconfig.sh b/hepix/commands/amiconfig.sh index 940d853..63355d9 100644 --- a/hepix/commands/amiconfig.sh +++ b/hepix/commands/amiconfig.sh @@ -4,6 +4,12 @@ AMICONFIG="/usr/sbin/amiconfig" AMILOCK="/var/lock/subsys/amiconfig" AMISETUP="/etc/sysconfig/amiconfig" +# Where to look for injected extra user-data +AMI_EXTRA_USER_DATA="/cernvm/extra-user-data" + +# A lock file to signal appended user-data +AMI_DONE_EXTRA_APPENDED='/var/lib/amiconfig/extra-appended.done' + # Maximum number of retries when attempting to retrieve user-data AMI_DOWNLOAD_RETRIES=2 @@ -28,6 +34,7 @@ fi RetrieveUserData() { if [ "$AMICONFIG_CONTEXT_URL" != '' ] ; then + # If context URL is found, it's from the environment. We must fill manually # the "local copy", where applicable $LOGGER "Won't check for new URLs: found from environment: $AMICONFIG_CONTEXT_URL" @@ -65,16 +72,12 @@ RetrieveUserData() { fi else + RetrieveUserDataCvmOnline || RetrieveUserDataEC2 || RetrieveUserDataCloudStack || RetrieveUserDataGCE - if [ $? == 1 ] ; then - if [ ! -f /cernvm/extra-user-data ]; then - $LOGGER "No user-data can be retrieved from any location" - return 1 - else - cat /cernvm/extra-user-data > "$AMICONFIG_LOCAL_USER_DATA" - chmod 600 "$AMICONFIG_LOCAL_USER_DATA" - fi + if [ $? != 0 ] ; then + $LOGGER "No standard user-data can be retrieved from any standard source: we are going to check for extra injected user data" fi + fi # At this point, user-data is available locally. Let's uncompress it if needed @@ -91,14 +94,67 @@ RetrieveUserData() { fi fi - if [ -f /cernvm/extra-user-data ]; then - cat /cernvm/extra-user-data >> "$AMICONFIG_LOCAL_USER_DATA" + # Now we retrieve the extra user data (which is never compressed) and append it, if it exists + if [ ! -e "$AMI_DONE_EXTRA_APPENDED" ] ; then + RetrieveUserDataInjected && touch "$AMI_DONE_EXTRA_APPENDED" + else + $LOGGER "Extra injected user-data already appended, skipping" + fi + + # Hard failure only if no user-data can be found either from standard location and extra injected path + if [ ! -r "$AMICONFIG_LOCAL_USER_DATA" ] ; then + return 1 fi return 0 } +# Retrieve injected user data. Injected user-data does not exclude other +# user-data sources, but it is always appended to existing user-data. In some +# cases the injected extra user-data can be the only source: this case is +# appropriately dealt with. Returns 1 on failure, 0 if found +RetrieveUserDataInjected() { + + # Fall back to this path if no existing user-data is there already + LOCAL_USER_DATA='/var/lib/amiconfig/2007-12-15' + + if [ -r "$AMI_EXTRA_USER_DATA" ] ; then + + $LOGGER "Extra injected configuration: additional user-data found at $AMI_EXTRA_USER_DATA" + + if [ "$AMICONFIG_CONTEXT_URL" == '' ] ; then + + $LOGGER "Extra injected configuration: this is the only user-data source available" + + # This is the only user-data! Set variables properly and save permanently + mkdir -p "$LOCAL_USER_DATA" + export AMICONFIG_CONTEXT_URL="file:$LOCAL_USER_DATA" + export AMICONFIG_LOCAL_USER_DATA="${LOCAL_USER_DATA}/user-data" + echo "export AMICONFIG_CONTEXT_URL=$AMICONFIG_CONTEXT_URL" > $AMISETUP + + # Add a newline at the end (might be missing) + echo '' > "$AMICONFIG_LOCAL_USER_DATA" + + else + + $LOGGER "Extra injected configuration: appending configuration to current user-data at $AMICONFIG_LOCAL_USER_DATA" + + fi + + # Append extra injected user-data + cat "$AMI_EXTRA_USER_DATA" >> "$AMICONFIG_LOCAL_USER_DATA" + chmod 0600 "$AMICONFIG_LOCAL_USER_DATA" + + return 0 # success (found) + + fi + + $LOGGER "Extra injected configuration: no injected user-data found at $AMI_EXTRA_USER_DATA" + return 1 # failure (not found) + +} + # Tries to check for the user-data file left there from CernVM Online. # Returns 0 on success, 1 on failure RetrieveUserDataCvmOnline() { From cc8d4f2cb26335364e4aa94565ae2a923305f5a6 Mon Sep 17 00:00:00 2001 From: Dario Berzano Date: Tue, 6 May 2014 14:38:52 +0200 Subject: [PATCH 2/7] Var to use local metadata server no longer needed --- hepix/commands/amiconfig.sh | 2 -- rpath/amiconfig/instancedata.py | 14 -------------- 2 files changed, 16 deletions(-) diff --git a/hepix/commands/amiconfig.sh b/hepix/commands/amiconfig.sh index 63355d9..4e096b0 100644 --- a/hepix/commands/amiconfig.sh +++ b/hepix/commands/amiconfig.sh @@ -538,8 +538,6 @@ Main() { case "$MODE" in user) RunUserDataScript before - # Hack: read meta-data from HTTP, user-data locally to include extra user data - export AMICONFIG_EC2_USER_DATA_URL="file:$(dirname ${AMICONFIG_LOCAL_USER_DATA})" $AMICONFIG 2>&1 #| $PIPELOGGER RunUserDataScript after mkdir -p `dirname "$AMI_DONE_USER"` diff --git a/rpath/amiconfig/instancedata.py b/rpath/amiconfig/instancedata.py index e307204..5292bb2 100644 --- a/rpath/amiconfig/instancedata.py +++ b/rpath/amiconfig/instancedata.py @@ -42,20 +42,6 @@ def open(self, path): return results def read(self, path): - ec2_user_data_url = os.getenv("AMICONFIG_EC2_USER_DATA_URL") - if ec2_user_data_url is not None: - try: - results = urllib.urlopen('%s/user-data' % (ec2_user_data_url)) - except Exception, e: - raise EC2DataRetrievalError, '[Errno %s] %s' % (e.errno, e.strerror) - if results.headers.gettype() == 'text/html': - # Eucalyptus returns text/html and no Server: header - # We want to protect ourselves from HTTP servers returning real - # HTML, so let's hope at least they're conformant and return a - # Server: header - if 'server' in results.headers: - raise EC2DataRetrievalError, '%s' % results.read() - return results.read() return self.open(path).read() def getUserData(self): From 125889cb9dcf6f01c74fe57386587b440bd06094 Mon Sep 17 00:00:00 2001 From: Dario Berzano Date: Tue, 6 May 2014 15:05:21 +0200 Subject: [PATCH 3/7] Honor local user-data cache Also allows to retrieve the user-data separately than to "patch" it before running amiconfig. Common use cases: - do not stress metadata server by sending a single request - append an "extra" user data retrieved separately (e.g. with injection) --- rpath/amiconfig/instancedata.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rpath/amiconfig/instancedata.py b/rpath/amiconfig/instancedata.py index 5292bb2..25afe35 100644 --- a/rpath/amiconfig/instancedata.py +++ b/rpath/amiconfig/instancedata.py @@ -45,6 +45,17 @@ def read(self, path): return self.open(path).read() def getUserData(self): + + # Read user-data from cache if possible + user_data_cache = os.getenv("AMICONFIG_LOCAL_USER_DATA") + if user_data_cache is not None and user_data_cache[0] == '/': + try: + with open(user_data_cache) as f: + return f.read() + except IOError, e: + pass + + # Fall back to standard read return self.read('user-data') def getAMIId(self): From e076fa3ddfefdba858074797c0f69f8c94c8c82a Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Wed, 7 May 2014 21:40:53 +0200 Subject: [PATCH 4/7] contextualize from hdd --- hepix/extra/hepix_functions | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/hepix/extra/hepix_functions b/hepix/extra/hepix_functions index 548644d..36951d9 100644 --- a/hepix/extra/hepix_functions +++ b/hepix/extra/hepix_functions @@ -83,21 +83,25 @@ getResourcesFromCD() { context_disk_found="" mount_dir=$(mktemp -d) - for dev in /dev/sr{0,1} /dev/{xv,v,h,s}d{c,d,b}; do + for dev in /dev/sr{0,1} /dev/{xv,v,h,s}d{c,d,b}{,1}; do if [ -e "${dev}" ] ; then rdisk=$dev - logger -t $0 "Attempting to mount contextualization image from $rdisk" - mount -t iso9660 ${rdisk} $mount_dir > /dev/null 2>&1 - if [ 0 -eq $? ]; then - if [ -e "$mount_dir/prolog.sh" -o -e "$mount_dir/context.sh" ]; then - cp -r $mount_dir/* $CONTEXT_DIR/ - chmod 700 $CONTEXT_DIR - context_disk_found="indeed" - break - else - umount $mount_dir >/dev/null 2>&1 - fi - fi + fstype=$(blkid $rdisk | awk '{print $4}' | sed s/TYPE=// | tr -d \") + if [ "x$fstype" = "xiso9660" -o "x$fstype" = "xvfat" ]; then + logger -t $0 "Attempting to mount contextualization image from $rdisk" + + mount -o ro ${rdisk} $mount_dir > /dev/null 2>&1 + if [ 0 -eq $? ]; then + if [ -e "$mount_dir/prolog.sh" -o -e "$mount_dir/context.sh" ]; then + cp -r $mount_dir/* $CONTEXT_DIR/ + chmod 700 $CONTEXT_DIR + context_disk_found="indeed" + break + else + umount $mount_dir >/dev/null 2>&1 + fi + fi + fi fi done if [ -z "${rdisk}" ]; then From fa25d9f0e1697b1875c89a775c42ebd524c632a7 Mon Sep 17 00:00:00 2001 From: Dario Berzano Date: Fri, 9 May 2014 19:54:07 +0200 Subject: [PATCH 5/7] Fix for wrong parsing of blkid output --- hepix/extra/hepix_functions | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hepix/extra/hepix_functions b/hepix/extra/hepix_functions index 36951d9..0be20ef 100644 --- a/hepix/extra/hepix_functions +++ b/hepix/extra/hepix_functions @@ -86,7 +86,11 @@ getResourcesFromCD() { for dev in /dev/sr{0,1} /dev/{xv,v,h,s}d{c,d,b}{,1}; do if [ -e "${dev}" ] ; then rdisk=$dev - fstype=$(blkid $rdisk | awk '{print $4}' | sed s/TYPE=// | tr -d \") + + # Extract filesystem type of device + fstype=$( blkid $rdisk | grep -o -E 'TYPE="[0-9a-zA-Z]+' ) + fstype=${fstype:6} + if [ "x$fstype" = "xiso9660" -o "x$fstype" = "xvfat" ]; then logger -t $0 "Attempting to mount contextualization image from $rdisk" From 230f488ba811c29883530d3b25d36b6d0744e00e Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Tue, 13 May 2014 20:17:18 +0200 Subject: [PATCH 6/7] fix recogition of vfat context image --- hepix/extra/hepix_functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hepix/extra/hepix_functions b/hepix/extra/hepix_functions index 0be20ef..2d2a61b 100644 --- a/hepix/extra/hepix_functions +++ b/hepix/extra/hepix_functions @@ -88,7 +88,7 @@ getResourcesFromCD() { rdisk=$dev # Extract filesystem type of device - fstype=$( blkid $rdisk | grep -o -E 'TYPE="[0-9a-zA-Z]+' ) + fstype=$( blkid $rdisk | sed -e 's/SEC_TYPE//' | grep -o -E 'TYPE="[0-9a-zA-Z]+' ) fstype=${fstype:6} if [ "x$fstype" = "xiso9660" -o "x$fstype" = "xvfat" ]; then From e532cb1954fa9b1c66cec29ef1c350d1e567379c Mon Sep 17 00:00:00 2001 From: Jakob Blomer Date: Wed, 14 May 2014 08:05:07 +0200 Subject: [PATCH 7/7] fix default modules --- rpath/amiconfig/instancedata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rpath/amiconfig/instancedata.py b/rpath/amiconfig/instancedata.py index 25afe35..573ffd0 100644 --- a/rpath/amiconfig/instancedata.py +++ b/rpath/amiconfig/instancedata.py @@ -50,8 +50,7 @@ def getUserData(self): user_data_cache = os.getenv("AMICONFIG_LOCAL_USER_DATA") if user_data_cache is not None and user_data_cache[0] == '/': try: - with open(user_data_cache) as f: - return f.read() + return open(user_data_cache).read() except IOError, e: pass