From 8dea33609fe72968c02b10772642aa414360b861 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Mon, 29 Aug 2016 12:25:19 -0400 Subject: [PATCH 01/10] Remove trailing whitespace --- bin/geni-sync-wireless | 90 +++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index dda4f3b9..39b3549c 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -46,26 +46,26 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base def parse_args(argv): - parser = optparse.OptionParser(usage="Synchronize ORBIT and GENI CH " + + parser = optparse.OptionParser(usage="Synchronize ORBIT and GENI CH " + "sense of projects/groups and members") - parser.add_option("--holdingpen_group", + parser.add_option("--holdingpen_group", help="Name of ORBIT 'holding pen' that is the primary"+\ - " group for all GENI users in wimax-enabled projects [default: %default]", + " group for all GENI users in wimax-enabled projects [default: %default]", default="geni-HOLDINGPEN") - parser.add_option("--holdingpen_admin", + parser.add_option("--holdingpen_admin", help="GENI username of admin of ORBIT 'holding pen' [default: %default]", default="agosain") - parser.add_option("--project", help="specific project name to sync", + parser.add_option("--project", help="specific project name to sync", default=None) parser.add_option("--user", help="specific username to sync", default=None) - parser.add_option("--cleanup", - help="delete obsolete groups and group memberships [default: %default]", + parser.add_option("--cleanup", + help="delete obsolete groups and group memberships [default: %default]", dest='cleanup', action='store_true', default=False) - parser.add_option("--settings_file", + parser.add_option("--settings_file", help="location of settings.php file containing db_dsn (database URL) variable [default: %default]", default="/etc/geni-ch/settings.php") - parser.add_option("-v", "--verbose", help="Print verbose debug info", + parser.add_option("-v", "--verbose", help="Print verbose debug info", dest = 'verbose', action='store_true', default=False) @@ -78,7 +78,7 @@ def parse_args(argv): return options,args -# Manaager to manage synchronize between ORBIT groups/users and GENI +# Manaager to manage synchronize between ORBIT groups/users and GENI # CH wimax-enabled projects and members class WirelessProjectManager: @@ -98,15 +98,15 @@ class WirelessProjectManager: self._session = self._session_class() self.PROJECT_TABLE = Table('pa_project', self._metadata, autoload=True) - self.PROJECT_ATTRIBUTE_TABLE = Table('pa_project_attribute', + self.PROJECT_ATTRIBUTE_TABLE = Table('pa_project_attribute', self._metadata, autoload=True) - self.PROJECT_MEMBER_TABLE = Table('pa_project_member', + self.PROJECT_MEMBER_TABLE = Table('pa_project_member', self._metadata, autoload=True) self.MEMBER_ATTRIBUTE_TABLE = Table('ma_member_attribute', self._metadata, autoload=True) self.SSH_KEY_TABLE = Table('ma_ssh_key', self._metadata, autoload=True) - self.SERVICE_REGISTRY_TABLE = Table('service_registry', + self.SERVICE_REGISTRY_TABLE = Table('service_registry', self._metadata, autoload=True) self.holdingpen_group_description = "GENI ORBIT MEMBER HOLDINGPEN" @@ -155,7 +155,7 @@ class WirelessProjectManager: db_url = line.split('\'')[1].replace('pgsql', 'postgresql') return db_url self.error("No $db_dsn entry in settings file") - + # Print error and exit def error(self, msg): syslog(msg); sys.exit() @@ -165,11 +165,11 @@ class WirelessProjectManager: return member_info['displayName'] elif 'first_name' in member_info and \ 'last_name' in member_info: - return "%s %s" % (member_info['first_name'], + return "%s %s" % (member_info['first_name'], member_info['last_name']) else: return member_info['email_address'] - + # Turn GENI name to ORBIT name (add geni- prefix) def to_orbit_name(self, name): return "geni-%s" % name @@ -209,7 +209,7 @@ class WirelessProjectManager: # Make sure the 'holding pen' group and admin exist # Make sure all members of wimax-enabled projects exist in ORBIT # Make sure all wimax-enabled projects exist as ORBIT groups - # Make sure membership in wimax-enabled projects leads to + # Make sure membership in wimax-enabled projects leads to # membership in ORBIT groups # Make sure project lead on wimax-enabled projects translates to # admin in ORBIT group @@ -230,7 +230,7 @@ class WirelessProjectManager: # Grab members in wimax-enabled projects self.get_geni_members() - # Remove disabled members from projects and members + # Remove disabled members from projects and members # (unless member is lead) self.remove_disabled_members() @@ -263,7 +263,7 @@ class WirelessProjectManager: # Make sure the admins of orbit groups match the leads of GENI projects self.ensure_project_leads_are_group_admins() - # If we're doing cleanup, + # If we're doing cleanup, # delete group members who aren't project members # delete groups that aren't GENI projects # disable any users not in any GENI project @@ -322,7 +322,7 @@ class WirelessProjectManager: holdingpen_admin_info['email_address'], holdingpen_admin_info['last_name'], holdingpen_admin_ssh_keys, - self.holdingpen_group_description, + self.holdingpen_group_description, user_irodsname) syslog("Creating holdingpen admin: %s" % \ holdingpen_admin_username) @@ -334,7 +334,7 @@ class WirelessProjectManager: # If not, create and place in holdingpen group as their primary group # The holdingpen admin is in the list of geni members, but don't need # to create his account: should already be there - def ensure_project_members_exist(self): + def ensure_project_members_exist(self): for member_id, member_info in self._geni_members.items(): username = member_info['username'] if username == self._options.holdingpen_admin: @@ -349,12 +349,12 @@ class WirelessProjectManager: member_pretty_name = self.get_pretty_name(member_info) syslog("Creating ORBIT user: %s" % orbit_username) first_name = "" - if 'first_name' in member_info: + if 'first_name' in member_info: first_name = member_info['first_name'] elif 'email_address' in member_info: first_name = member_info['email_address'] last_name = "" - if 'last_name' in member_info: + if 'last_name' in member_info: last_name = member_info['last_name'] irodsname = None if 'irods_username' in member_info: @@ -371,10 +371,10 @@ class WirelessProjectManager: irodsname) self._orb.saveUser(ldif_text) - - # Make sure all wimax-enabled GENI projects have a corresponding + + # Make sure all wimax-enabled GENI projects have a corresponding # ORBIT group - def ensure_projects_exist(self): + def ensure_projects_exist(self): for project_id, project_info in self._geni_projects.items(): project_name = project_info['project_name'] project_description = project_info['project_description'] @@ -384,10 +384,10 @@ class WirelessProjectManager: lead_id = project_info['lead_id'] lead_username = self._geni_members[lead_id]['username'] orbit_lead_username = self.to_orbit_name(lead_username) - ldif_text = self._orb.ldif_for_group(orbit_group_name, + ldif_text = self._orb.ldif_for_group(orbit_group_name, project_description) ldif_text = ldif_text + \ - self._orb.ldif_for_group_admin(orbit_group_name, + self._orb.ldif_for_group_admin(orbit_group_name, orbit_lead_username, self._options.holdingpen_group) self._orb.saveUser(ldif_text) @@ -401,7 +401,7 @@ class WirelessProjectManager: # Make sure all members of wimax-enabledf GENI projects are membes # of the corresponding ORBIT group # Enable all users that are members of a non-holdingpen group - def ensure_project_members_in_groups(self): + def ensure_project_members_in_groups(self): users_to_enable = set() for project_id, project_info in self._geni_projects.items(): project_name = project_info['project_name'] @@ -415,7 +415,7 @@ class WirelessProjectManager: geni_username = member_info['username'] orbit_username = self.to_orbit_name(geni_username) if orbit_username not in group_info['users']: - syslog("Adding user %s to group %s" % (orbit_username, + syslog("Adding user %s to group %s" % (orbit_username, orbit_group_name)) self._orb.add_user_to_group(orbit_group_name, orbit_username) users_to_enable.add(orbit_username) @@ -426,7 +426,7 @@ class WirelessProjectManager: self._orb.enable_user(user_to_enable) # Make sure the lead of the project is the corresponding group admin - def ensure_project_leads_are_group_admins(self): + def ensure_project_leads_are_group_admins(self): for project_id, project_info in self._geni_projects.items(): project_name = project_info['project_name'] orbit_group_name = self.to_orbit_name(project_name) @@ -441,7 +441,7 @@ class WirelessProjectManager: # Delete members of a group that aren't members of corresponding project # Keep list of users removed from groups - def delete_group_members_not_in_project(self): + def delete_group_members_not_in_project(self): for group_name, group_info in self._orbit_groups.items(): geni_project_name = self.to_geni_name(group_name) if self._project and geni_project_name != self._project: continue @@ -454,7 +454,7 @@ class WirelessProjectManager: if geni_member_id in self._geni_members] else: # No GENI project, remove all group members - geni_project_members = [] + geni_project_members = [] for orbit_username in group_info['users']: geni_username = self.to_geni_name(orbit_username) if geni_username not in geni_project_members: @@ -467,7 +467,7 @@ class WirelessProjectManager: # Delete groups that don't correspond to projects # Keep a list of deleted groups - def delete_groups_without_projects(self): + def delete_groups_without_projects(self): for group_name, group_info in self._orbit_groups.items(): geni_project_name = self.to_geni_name(group_name) if self._project and geni_project_name != self._project: continue @@ -479,10 +479,10 @@ class WirelessProjectManager: self._deleted_orbit_groups.append(group_name) # Disable users who are only in the ORBIT holdingpen group - # Note: we've deleted some projects at this point, so + # Note: we've deleted some projects at this point, so # we mean users who are in at least one recently deleted group # but no other non-deleted groups - def disable_users_in_no_project(self): + def disable_users_in_no_project(self): for orbit_username in self._orbit_users: geni_username = self.to_geni_name(orbit_username) user_in_some_deleted_group = False @@ -522,10 +522,10 @@ class WirelessProjectManager: projects = {} # Get all the WIMAX-enabled projects - query = self._session.query(self.PROJECT_TABLE.c.lead_id, - self.PROJECT_TABLE.c.project_name, - self.PROJECT_TABLE.c.project_id, - self.PROJECT_TABLE.c.project_purpose) + query = self._session.query(self.PROJECT_TABLE.c.lead_id, + self.PROJECT_TABLE.c.project_name, + self.PROJECT_TABLE.c.project_id, + self.PROJECT_TABLE.c.project_purpose) query = query.filter(self.PROJECT_TABLE.c.project_id == \ self.PROJECT_ATTRIBUTE_TABLE.c.project_id) query = query.filter(self.PROJECT_ATTRIBUTE_TABLE.c.name == \ @@ -563,7 +563,7 @@ class WirelessProjectManager: for row in member_rows: projects[row.project_id]['members'].append(row.member_id) - # Don't return any projects with no members + # Don't return any projects with no members # (if we're filtering by user) for project_id, project_info in projects.items(): if len(project_info['members']) == 0: @@ -605,18 +605,18 @@ class WirelessProjectManager: query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.member_id.in_(\ member_ids)) query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.name.in_(\ - ["username", 'member_enabled', "first_name", "email_address", + ["username", 'member_enabled', "first_name", "email_address", "last_name", "displayName", 'wimax_username'])) - + member_rows = query.all() for row in member_rows: - if row.member_id not in members: + if row.member_id not in members: members[row.member_id] = {} members[row.member_id][row.name] = row.value # Grab SSH keys for all members - for member_id, member_info in members.items(): + for member_id, member_info in members.items(): member_info['ssh_keys']=[] query = self._session.query(self.SSH_KEY_TABLE) query = query.filter(self.SSH_KEY_TABLE.c.member_id.in_(\ From 8dcc7f04904b52ebc16d4893d58c815409320a37 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Mon, 29 Aug 2016 14:34:31 -0400 Subject: [PATCH 02/10] Switch from optparse to argparse Change geni-sync-wireless from optparse to argparse for command line options. optparse is deprecated, argparse is the replacement going forward. --- bin/geni-sync-wireless | 58 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index 39b3549c..e60ae927 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -37,7 +37,7 @@ import datetime import logging import xml.dom.minidom -import optparse +import argparse import sys from syslog import syslog from portal_utils.orbit_interface import ORBIT_Interface @@ -46,37 +46,37 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base def parse_args(argv): - parser = optparse.OptionParser(usage="Synchronize ORBIT and GENI CH " + - "sense of projects/groups and members") - parser.add_option("--holdingpen_group", - help="Name of ORBIT 'holding pen' that is the primary"+\ - " group for all GENI users in wimax-enabled projects [default: %default]", - default="geni-HOLDINGPEN") - parser.add_option("--holdingpen_admin", - help="GENI username of admin of ORBIT 'holding pen' [default: %default]", - default="agosain") - parser.add_option("--project", help="specific project name to sync", - default=None) - parser.add_option("--user", help="specific username to sync", default=None) - parser.add_option("--cleanup", - help="delete obsolete groups and group memberships [default: %default]", - dest='cleanup', action='store_true', - default=False) - parser.add_option("--settings_file", - help="location of settings.php file containing db_dsn (database URL) variable [default: %default]", - default="/etc/geni-ch/settings.php") - parser.add_option("-v", "--verbose", help="Print verbose debug info", - dest = 'verbose', action='store_true', - default=False) - - options,args = parser.parse_args() + desc = 'Synchronize ORBIT and GENI CH sense of projects/groups and members' + parser = argparse.ArgumentParser(description=desc) + parser.add_argument("--holdingpen_group", + help="Name of ORBIT 'holding pen' that is the primary"+\ + " group for all GENI users in wimax-enabled projects [default: %(default)s]", + default="geni-HOLDINGPEN") + parser.add_argument("--holdingpen_admin", + help="GENI username of admin of ORBIT 'holding pen' [default: %(default)s]", + default="agosain") + parser.add_argument("--project", help="specific project name to sync", + default=None) + parser.add_argument("--user", help="specific username to sync", default=None) + parser.add_argument("--cleanup", + help="delete obsolete groups and group memberships [default: %(default)s]", + dest='cleanup', action='store_true', + default=False) + parser.add_argument("--settings_file", + help="location of settings.php file containing db_dsn (database URL) variable [default: %(default)s]", + default="/etc/geni-ch/settings.php") + parser.add_argument("-v", "--verbose", help="Print verbose debug info", + dest = 'verbose', action='store_true', + default=False) + + args = parser.parse_args() # User and project options are mutually exclusive - if options.project and options.user: + if args.project and args.user: syslog( "Only one of --project, --user allowed") sys.exit() - return options,args + return args # Manaager to manage synchronize between ORBIT groups/users and GENI # CH wimax-enabled projects and members @@ -658,9 +658,9 @@ class WirelessProjectManager: def main(): - options, args = parse_args(sys.argv) + args = parse_args(sys.argv) - wpm = WirelessProjectManager(options) + wpm = WirelessProjectManager(args) wpm.synchronize() From c7caa735ab5b2491c4ffcaca44b76451e1e6fc47 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Mon, 12 Sep 2016 16:40:04 -0400 Subject: [PATCH 03/10] Sync wireless via clearinghouse API Start using the clearinghouse API to get information about wireless projects and users. Stop using the database for project information in geni-sync-wireless. --- bin/geni-sync-wireless | 231 ++++++++++++++++++++++++++++++++--------- 1 file changed, 180 insertions(+), 51 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index e60ae927..09f49ccb 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -45,6 +45,14 @@ from sqlalchemy import * from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base +sys.path.append('/usr/share/geni-ch/gcf/src') + +try: + from gcf.geni.util.secure_xmlrpc_client import make_client +except: + raise + + def parse_args(argv): desc = 'Synchronize ORBIT and GENI CH sense of projects/groups and members' parser = argparse.ArgumentParser(description=desc) @@ -65,6 +73,13 @@ def parse_args(argv): parser.add_argument("--settings_file", help="location of settings.php file containing db_dsn (database URL) variable [default: %(default)s]", default="/etc/geni-ch/settings.php") + parser.add_argument('-c', '--certificate', + help='certificate of user performing action') + parser.add_argument('-k', '--key', + help='private key of user performing action') + parser.add_argument('-u', '--url', + help='Clearinghouse base URL [default: %(default)s]', + default='https://ch.geni.net') parser.add_argument("-v", "--verbose", help="Print verbose debug info", dest = 'verbose', action='store_true', default=False) @@ -78,6 +93,88 @@ def parse_args(argv): return args + +class GeniException(Exception): + pass + + +class GeniResponse(object): + CODE = 'code' + VALUE = 'value' + OUTPUT = 'output' + SUCCESS = 0 + + @classmethod + def check(cls, response): + if GeniResponse.CODE not in response: + raise GeniException('Invalid response, no "code".') + if response[GeniResponse.CODE] != GeniResponse.SUCCESS: + msg = 'Server error %d' % (response[GeniResponse.CODE]) + if GeniResponse.OUTPUT in response: + msg += ': %s' % (response[GeniResponse.OUTPUT]) + raise GeniException(msg) + + +class ProjectAttribute(object): + NAME = 'name' + ENABLE_WIMAX = 'enable_wimax' + # not sure we need wimax group, but just in case... + WIMAX_GROUP = 'wimax_group_name' + + +class Project(object): + SA_UID = 'PROJECT_UID' + SA_LEAD = '_GENI_PROJECT_OWNER' + SA_DESCRIPTION = 'PROJECT_DESCRIPTION' + SA_NAME = 'PROJECT_NAME' + SA_URN = 'PROJECT_URN' + + @classmethod + def create(cls, map): + p = Project() + p.id = map[Project.SA_UID] + p.lead_id = map[Project.SA_LEAD] + p.project_description = map[Project.SA_DESCRIPTION] + p.project_name = map[Project.SA_NAME] + p.urn = map[Project.SA_URN] + return p + + def __init__(self): + self.members = [] + + def __getitem__(self, key): + print "key is %r" % (key) + if not hasattr(self, key): + raise KeyError(key) + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + print "setitem(%r, %r)" % (key, value) + + def __delitem__(self, key): + delattr(self, key) + + def keys(self): + return self.__dict__.keys() + + +class GENI(object): + class MA(object): + EMAIL = 'MEMBER_EMAIL' + FIRST_NAME = 'MEMBER_FIRSTNAME' + LAST_NAME = 'MEMBER_LASTNAME' + DISPLAY_NAME = '_GENI_MEMBER_DISPLAYNAME' + USERNAME = 'MEMBER_USERNAME' + WIMAX_USERNAME = '_GENI_WIMAX_USERNAME' + FieldMap = {EMAIL: 'email_address', + FIRST_NAME: 'first_name', + LAST_NAME: 'last_name', + DISPLAY_NAME: 'displayName', + USERNAME: 'username', + WIMAX_USERNAME: 'wimax_username'} + + # Manaager to manage synchronize between ORBIT groups/users and GENI # CH wimax-enabled projects and members class WirelessProjectManager: @@ -88,6 +185,9 @@ class WirelessProjectManager: self._project = self._options.project self._cleanup = self._options.cleanup self._user = self._options.user + self._url = self._options.url + self._certificate = self._options.certificate + self._key = self._options.key self._db_url = self.find_database_url() self._db = create_engine(self._db_url) @@ -515,61 +615,90 @@ class WirelessProjectManager: return project_info return None + def make_project_dict(self, sa_project): + """Extract information from the slice authority (SA) response + pertaining to a project. Return a dict comprising the information + needed for other functions later in the wireless sync process. + """ + return dict(lead_id=sa_project[Project.SA_LEAD], + members=[], + project_description=sa_project[Project.SA_DESCRIPTION], + project_name=sa_project[Project.SA_NAME], + urn=sa_project[Project.SA_URN]) + + def is_wimax_project(self, xmlrpc_client, sa_project): + is_wimax = False + credentials = [] + project_id = sa_project[Project.SA_UID] + project_urn = sa_project[Project.SA_URN] + opt_match = {Project.SA_UID: [project_id]} + options = dict(match=opt_match) + response = xmlrpc_client.lookup_project_attributes(project_urn, + credentials, + options) + # Check for errors from server + GeniResponse.check(response) + attribs = response[GeniResponse.VALUE] + for attrib in attribs: + if attrib[ProjectAttribute.NAME] == ProjectAttribute.ENABLE_WIMAX: + is_wimax = True + break + return is_wimax + + def get_project_members(self, client, project_info, username): + credentials = [] + options = {} + urn = project_info['urn'] + response = client.lookup_project_members(urn, credentials, options) + # Check for errors from server + GeniResponse.check(response) + members = response[GeniResponse.VALUE] + urn_suffix = '' + if username: + urn_suffix = '+user+%s' % (str(username)) + filtered_members = [m['PROJECT_MEMBER_UID'] for m in members + if m['PROJECT_MEMBER'].endswith(urn_suffix)] + return filtered_members + + def get_wimax_projects(self, xmlrpc_client, project_name=None, + user_name=None): + credentials = [] + opt_match = dict(PROJECT_EXPIRED=False) + # If --project is specified add it to opt_match + if project_name: + opt_match[Project.SA_NAME] = project_name + opt_filter = [Project.SA_UID, Project.SA_LEAD, Project.SA_DESCRIPTION, + Project.SA_NAME, Project.SA_URN] + options = dict(match=opt_match, filter=opt_filter) + # options = dict(match=opt_match) + response = xmlrpc_client.lookup('PROJECT', credentials, options) + # Check for errors from server + GeniResponse.check(response) + all_projects = response[GeniResponse.VALUE] + wimax_projects = {p[Project.SA_UID]: self.make_project_dict(p) + for p in all_projects.values() + if self.is_wimax_project(xmlrpc_client, p)} + # Now add users to the wimax_projects + # TODO: if --user was specified, only include that username + # Do this by checking if the user's urn .endswith(username) + # if self._username and not urn.endswith(self.__username): + # contine + + for proj in wimax_projects.values(): + proj['members'] = self.get_project_members(xmlrpc_client, proj, + user_name) + return wimax_projects + # Grab project info [indexed by project id] for all wimax-enabled projects # Only single project for --project option # Only projects to which given users belongs for --user option def get_geni_projects(self): - projects = {} - - # Get all the WIMAX-enabled projects - query = self._session.query(self.PROJECT_TABLE.c.lead_id, - self.PROJECT_TABLE.c.project_name, - self.PROJECT_TABLE.c.project_id, - self.PROJECT_TABLE.c.project_purpose) - query = query.filter(self.PROJECT_TABLE.c.project_id == \ - self.PROJECT_ATTRIBUTE_TABLE.c.project_id) - query = query.filter(self.PROJECT_ATTRIBUTE_TABLE.c.name == \ - 'enable_wimax') - query = query.filter(self.PROJECT_TABLE.c.expired == 'f') - if (self._project): - query = query.filter(self.PROJECT_TABLE.c.project_name == \ - self._project) - project_rows = query.all() - project_ids = [] - - - for row in project_rows: - project_ids.append(row.project_id) - projects[row.project_id] = { - 'lead_id' : row.lead_id, - 'project_name' : row.project_name, - 'project_description' : row.project_purpose, - 'members' : [] - } - - # Get all members of WIMAX-enabled projects - query = self._session.query(self.PROJECT_MEMBER_TABLE.c.member_id, - self.PROJECT_MEMBER_TABLE.c.project_id) - query = query.filter(self.PROJECT_MEMBER_TABLE.c.project_id.in_(\ - project_ids)) - if self._user: - query = query.filter(self.PROJECT_MEMBER_TABLE.c.member_id == \ - self.MEMBER_ATTRIBUTE_TABLE.c.member_id) - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.name == \ - 'username') - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.value == \ - self._user) - member_rows = query.all() - for row in member_rows: - projects[row.project_id]['members'].append(row.member_id) - - # Don't return any projects with no members - # (if we're filtering by user) - for project_id, project_info in projects.items(): - if len(project_info['members']) == 0: - del projects[project_id] - - self._geni_projects = projects + # TODO: Should get URL from service registry, not invent the URL + # by string appends + sa_url = self._url + '/SA' + sa_client = make_client(sa_url, self._key, self._certificate) + self._geni_projects = self.get_wimax_projects(sa_client, self._project, + self._user) # Grab info about all people in wimax projects def get_geni_members(self): From 75f11f141c843a30708fcedb9995a3fe3897577d Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 09:27:00 -0400 Subject: [PATCH 04/10] Sync wireless use more clearinghouse API Use the clearinghouse API to gather member info instead of diving into the database. --- bin/geni-sync-wireless | 124 ++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 52 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index 09f49ccb..29b22e7f 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -700,61 +700,81 @@ class WirelessProjectManager: self._geni_projects = self.get_wimax_projects(sa_client, self._project, self._user) - # Grab info about all people in wimax projects - def get_geni_members(self): + def holdingpen_admin_ids(self, ma_client): + # Get the UUID of the 'holdingpen_admin' + opt_match = {'MEMBER_USERNAME': self._options.holdingpen_admin} + opt_filter = ['MEMBER_UID'] + options = dict(match=opt_match, filter=opt_filter) + # pprint.pprint(options) + credentials = [] + response = ma_client.lookup_public_member_info(credentials, options) + # Check for errors from server + GeniResponse.check(response) + members = response[GeniResponse.VALUE] + result = set() + for info in members.values(): + result.add(info['MEMBER_UID']) + return list(result) - projects = self._geni_projects - members = {} + def get_ssh_keys(self, ma_client, members): + all_uuids = [v['MEMBER_UID'] for v in members.values()] + opt_match = {'_GENI_KEY_MEMBER_UID': all_uuids} + opt_filter = ['KEY_PUBLIC'] + options = dict(match=opt_match, filter=opt_filter) + credentials = [] + response = ma_client.lookup_keys(credentials, options) + # Check for errors from server + GeniResponse.check(response) + return response[GeniResponse.VALUE] + + def get_wimax_members(self, ma_client, projects): + all_uuids = set() + for proj_info in projects.values(): + all_uuids.update(proj_info['members']) + all_uuids.update(self.holdingpen_admin_ids(ma_client)) + opt_match = {'MEMBER_UID': list(all_uuids)} + opt_filter = [GENI.MA.EMAIL, GENI.MA.FIRST_NAME, 'MEMBER_LASTNAME', + 'MEMBER_USERNAME', '_GENI_MEMBER_DISPLAYNAME', + '_GENI_WIMAX_USERNAME', 'MEMBER_URN', '_GENI_MEMBER_ENABLED', + 'MEMBER_UID'] + options = dict(match=opt_match, filter=opt_filter) + # pprint.pprint(options) + credentials = [] + response = ma_client.lookup('MEMBER', credentials, options) + # Check for errors from server + GeniResponse.check(response) + members = response[GeniResponse.VALUE] + # Filter out disabled members + members = {k: v for k,v in members.iteritems() + if v['_GENI_MEMBER_ENABLED']} + for k in members.keys(): + members[k]['ssh_keys'] = [] + # Attach SSH public key to user_info['ssh_keys'] + ssh_keys = self.get_ssh_keys(ma_client, members) + for k, v in ssh_keys.iteritems(): + members[k]['ssh_keys'] = [] + for ssh_key in v: + # TODO: Why do the ssh keys have a '\n' at the end? + # Can ''.rstrip('\n') be used to *safely* strip it away? + members[k]['ssh_keys'].append(ssh_key['KEY_PUBLIC']) + # TODO: Map response keys to user_info keys (see above) + all_urns = members.keys() + for urn in all_urns: + for ma_key,new_key in GENI.MA.FieldMap.iteritems(): + members[urn][new_key] = members[urn].pop(ma_key) + uid = members[urn]['MEMBER_UID'] + members[uid] = members.pop(urn) + return members - # Get unique list of all member_ids over all projects - member_ids = set() - for proj_id, project_info in projects.items(): - for member_id in project_info['members']: - member_ids.add(member_id) - - # add the holdingpen admin, who may not be the member of any project - query = self._session.query(self.MEMBER_ATTRIBUTE_TABLE.c.member_id) - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.name == 'username') - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.value == \ - self._options.holdingpen_admin) - holdingpen_admin_rows = query.all() - for row in holdingpen_admin_rows: - member_ids.add(row.member_id) - - # Add the project leads - for proj_id, project_info in projects.items(): - lead_id = project_info['lead_id'] - member_ids.add(lead_id) - - # Turn set back into list, to grab all users with these member ID's - member_ids = list(member_ids) - - query = self._session.query(self.MEMBER_ATTRIBUTE_TABLE) - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.member_id.in_(\ - member_ids)) - query = query.filter(self.MEMBER_ATTRIBUTE_TABLE.c.name.in_(\ - ["username", 'member_enabled', "first_name", "email_address", - "last_name", "displayName", 'wimax_username'])) - - member_rows = query.all() - for row in member_rows: - if row.member_id not in members: - members[row.member_id] = {} - members[row.member_id][row.name] = row.value - - - # Grab SSH keys for all members - for member_id, member_info in members.items(): - member_info['ssh_keys']=[] - query = self._session.query(self.SSH_KEY_TABLE) - query = query.filter(self.SSH_KEY_TABLE.c.member_id.in_(\ - member_ids)) - key_rows = query.all() - for row in key_rows: - members[row.member_id]['ssh_keys'].append(row.public_key) - - self._geni_members = members + # Grab info about all people in wimax projects + def get_geni_members(self): + # TODO: Should get URL from service registry, not invent the URL + # by string appends + ma_url = self._url + '/MA' + ma_client = make_client(ma_url, self._key, self._certificate) + projects = self._geni_projects + self._geni_members = self.get_wimax_members(ma_client, projects) # Remove disabled members from projects and members # Unless the disabled member is the lead of project From 1f119408b58ea1b2bad00f14c3264daf9de17d90 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 11:27:13 -0400 Subject: [PATCH 05/10] Use service registry in sync wireless Locate the required services via the service registry instead of either inventing them on the fly or digging them out of the database. --- bin/geni-sync-wireless | 45 ++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index 29b22e7f..1cbe43f3 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -41,6 +41,7 @@ import argparse import sys from syslog import syslog from portal_utils.orbit_interface import ORBIT_Interface +import xmlrpclib from sqlalchemy import * from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.declarative import declarative_base @@ -78,8 +79,8 @@ def parse_args(argv): parser.add_argument('-k', '--key', help='private key of user performing action') parser.add_argument('-u', '--url', - help='Clearinghouse base URL [default: %(default)s]', - default='https://ch.geni.net') + help='Service registry URL [default: %(default)s]', + default='https://ch.geni.net:8444/SR') parser.add_argument("-v", "--verbose", help="Print verbose debug info", dest = 'verbose', action='store_true', default=False) @@ -160,6 +161,7 @@ class Project(object): class GENI(object): + class MA(object): EMAIL = 'MEMBER_EMAIL' FIRST_NAME = 'MEMBER_FIRSTNAME' @@ -174,6 +176,12 @@ class GENI(object): USERNAME: 'username', WIMAX_USERNAME: 'wimax_username'} + class SR(object): + SERVICE_TYPE_SA = 1 + SERVICE_TYPE_MA = 3 + SERVICE_TYPE_WIMAX_SITE = 10 + SERVICE_URL = 'SERVICE_URL' + # Manaager to manage synchronize between ORBIT groups/users and GENI # CH wimax-enabled projects and members @@ -211,7 +219,7 @@ class WirelessProjectManager: self.holdingpen_group_description = "GENI ORBIT MEMBER HOLDINGPEN" - self._base_orbit_url = self.get_orbit_base_url() + self._base_orbit_url = self.get_orbit_base_url(self._url) self._orb = ORBIT_Interface(self._base_orbit_url) # These are instance variables filled in during synchronize @@ -229,17 +237,20 @@ class WirelessProjectManager: self._deleted_orbit_groups = [] self._deleted_orbit_members = {} - # Return URL for ORBIT Delegated AM REST API from service registry - def get_orbit_base_url(self): - WIMAX_SITE = 10 - query = self._session.query(self.SERVICE_REGISTRY_TABLE) - query = query.filter(self.SERVICE_REGISTRY_TABLE.c.service_type == WIMAX_SITE) - rows = query.all() + def get_service_url(self, sr_url, service_type): + sr = xmlrpclib.ServerProxy(sr_url) + response = sr.get_services_of_type(service_type) + GeniResponse.check(response) + rows = response[GeniResponse.VALUE] if len(rows) == 0: - self.error("NO WIMAX_SITE (%s) defined in SERVICE_REGISTRY" % \ - WIMAX_SITE) - orbit_base_url = rows[0].service_url - return orbit_base_url + msg = 'No service with type (%r) defined in SERVICE_REGISTRY' + self.error(msg % service_type) + service_url = rows[0][GENI.SR.SERVICE_URL] + return service_url + + # Return URL for ORBIT Delegated AM REST API from service registry + def get_orbit_base_url(self, sr_url): + return self.get_service_url(sr_url, GENI.SR.SERVICE_TYPE_WIMAX_SITE) # Parse settings.php file and extract db_dsn value def find_database_url(self): @@ -693,9 +704,7 @@ class WirelessProjectManager: # Only single project for --project option # Only projects to which given users belongs for --user option def get_geni_projects(self): - # TODO: Should get URL from service registry, not invent the URL - # by string appends - sa_url = self._url + '/SA' + sa_url = self.get_service_url(self._url, GENI.SR.SERVICE_TYPE_SA) sa_client = make_client(sa_url, self._key, self._certificate) self._geni_projects = self.get_wimax_projects(sa_client, self._project, self._user) @@ -769,9 +778,7 @@ class WirelessProjectManager: # Grab info about all people in wimax projects def get_geni_members(self): - # TODO: Should get URL from service registry, not invent the URL - # by string appends - ma_url = self._url + '/MA' + ma_url = self.get_service_url(self._url, GENI.SR.SERVICE_TYPE_MA) ma_client = make_client(ma_url, self._key, self._certificate) projects = self._geni_projects self._geni_members = self.get_wimax_members(ma_client, projects) From 297076080ed4f23de0131571360499613196ca1a Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 12:59:57 -0400 Subject: [PATCH 06/10] Use service registry in sync wireless Use MA API to add a wimax username to a user's info. No longer use the database directly for this. --- bin/geni-sync-wireless | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index 1cbe43f3..c816fa7b 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -237,6 +237,10 @@ class WirelessProjectManager: self._deleted_orbit_groups = [] self._deleted_orbit_members = {} + def get_ma_client(self, sr_url): + ma_url = self.get_service_url(sr_url, GENI.SR.SERVICE_TYPE_MA) + return make_client(ma_url, self._key, self._certificate) + def get_service_url(self, sr_url, service_type): sr = xmlrpclib.ServerProxy(sr_url) response = sr.get_services_of_type(service_type) @@ -289,18 +293,20 @@ class WirelessProjectManager: def to_geni_name(self, name): return name[5:] def insert_wimax_username(self, member_id, member_info): + # NOTE: member_id is not used in this implementation, but was + # used when this method accessed the database directly. username = member_info['username'] + member_urn = member_info['MEMBER_URN'] name = 'wimax_username' value = self.to_orbit_name(username) self_asserted = False syslog("Setting wimax_username for %s to %s" % (username, value)) - ins = self.MEMBER_ATTRIBUTE_TABLE.insert().values( - member_id=member_id, - name=name, - value=value, - self_asserted=self_asserted) - result = self._session.execute(ins) - self._session.commit() + ma = self.get_ma_client(self._url) + credentials = [] + options = {} + response = ma.add_member_attribute(member_urn, name, value, + self_asserted, credentials, options) + GeniResponse.check(response) def ensure_wimax_username(self, member_id, member_info): """Ensure that the given member has a wimax_username set @@ -778,8 +784,7 @@ class WirelessProjectManager: # Grab info about all people in wimax projects def get_geni_members(self): - ma_url = self.get_service_url(self._url, GENI.SR.SERVICE_TYPE_MA) - ma_client = make_client(ma_url, self._key, self._certificate) + ma_client = self.get_ma_client(self._url) projects = self._geni_projects self._geni_members = self.get_wimax_members(ma_client, projects) From 2f246bb2c17bcb3315241817a7bf3fa21aeab6b0 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 13:37:28 -0400 Subject: [PATCH 07/10] Remove database related code from sync wireless All access is now by API, none by database. Remove the vestiges of the database access because it is no longer needed. --- bin/geni-sync-wireless | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index c816fa7b..e1ad1fec 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -42,9 +42,6 @@ import sys from syslog import syslog from portal_utils.orbit_interface import ORBIT_Interface import xmlrpclib -from sqlalchemy import * -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base sys.path.append('/usr/share/geni-ch/gcf/src') @@ -71,9 +68,6 @@ def parse_args(argv): help="delete obsolete groups and group memberships [default: %(default)s]", dest='cleanup', action='store_true', default=False) - parser.add_argument("--settings_file", - help="location of settings.php file containing db_dsn (database URL) variable [default: %(default)s]", - default="/etc/geni-ch/settings.php") parser.add_argument('-c', '--certificate', help='certificate of user performing action') parser.add_argument('-k', '--key', @@ -197,26 +191,6 @@ class WirelessProjectManager: self._certificate = self._options.certificate self._key = self._options.key - self._db_url = self.find_database_url() - self._db = create_engine(self._db_url) - self._session_class = sessionmaker(bind=self._db) - self._metadata = MetaData(self._db) - base = declarative_base() - base.metadata.create_all(self._db) - self._session = self._session_class() - - self.PROJECT_TABLE = Table('pa_project', self._metadata, autoload=True) - self.PROJECT_ATTRIBUTE_TABLE = Table('pa_project_attribute', - self._metadata, autoload=True) - self.PROJECT_MEMBER_TABLE = Table('pa_project_member', - self._metadata, autoload=True) - self.MEMBER_ATTRIBUTE_TABLE = Table('ma_member_attribute', - self._metadata, autoload=True) - self.SSH_KEY_TABLE = Table('ma_ssh_key', - self._metadata, autoload=True) - self.SERVICE_REGISTRY_TABLE = Table('service_registry', - self._metadata, autoload=True) - self.holdingpen_group_description = "GENI ORBIT MEMBER HOLDINGPEN" self._base_orbit_url = self.get_orbit_base_url(self._url) @@ -256,21 +230,6 @@ class WirelessProjectManager: def get_orbit_base_url(self, sr_url): return self.get_service_url(sr_url, GENI.SR.SERVICE_TYPE_WIMAX_SITE) - # Parse settings.php file and extract db_dsn value - def find_database_url(self): - data = None - try: - data = open(self._options.settings_file, 'r').read() - except Exception: - self.error("Error reading settings file: %s" % \ - self._options.settings_file) - lines = data.split('\n') - for line in lines: - if line.find("$db_dsn") == 0: - db_url = line.split('\'')[1].replace('pgsql', 'postgresql') - return db_url - self.error("No $db_dsn entry in settings file") - # Print error and exit def error(self, msg): syslog(msg); sys.exit() From 46983e8040046259b618d112c2d1900991599e5c Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 13:56:54 -0400 Subject: [PATCH 08/10] Update geni-sync-wireless man page Add new arguments to man page, remove outdated arguments. --- man/geni-sync-wireless.1 | 51 +++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/man/geni-sync-wireless.1 b/man/geni-sync-wireless.1 index 37d70629..f74d9736 100644 --- a/man/geni-sync-wireless.1 +++ b/man/geni-sync-wireless.1 @@ -1,4 +1,4 @@ -.TH GENI-SYNC-WIRELESS 1 "August 11, 2015" +.TH GENI-SYNC-WIRELESS 1 "September 13, 2016" .SH NAME geni-sync-wireless \- Synchronize ORBIT and GENI CH sense of projects/groups and members .SH SYNOPSIS @@ -8,22 +8,24 @@ geni-sync-wireless \- Synchronize ORBIT and GENI CH sense of projects/groups and [\fB--project \fIPROJECT\fR] [\fB--user \fIUSERNAME\fR] [\fB--cleanup\fR] -[\fB--settings_file \fISETTINGS_FILE\fR] +[\fB--key \fIKEY_FILE\fR] +[\fB--certificate \fICERTIFICATE_FILE\fR] +[\fB--url \fISERVICE_REGISTRY_URL\fR] .SH DESCRIPTION -Perform synchronization between GENI clearinghouse wireless-enabled projects +Perform synchronization between GENI clearinghouse wireless-enabled projects and ORBIT groups, and their corresponding leads and membership rosters. -A wireless-enabled project is one with the 'enable_wimax' attribute. -In 'cleanup' mode (invoked by --cleanup), groups and members are deleted +A wireless-enabled project is one with the `enable_wimax' attribute. +In `cleanup' mode (invoked by --cleanup), groups and members are deleted from ORBIT if they are not found in corresponding GENI CH wireless-enabled projects and members. Otherwise (non-cleanup mode) new projects and members are written but not deleted. -The intended use of this script is that it be run in 'cleanup' mode -periodically, say nightly. Otherwise, it should be run for specific -projects when the 'ORBIT sync' button on the wimax-enable.php page is pressed. +The intended use of this script is that it be run in `cleanup' mode +periodically, say nightly. Otherwise, it should be run for specific +projects when the `ORBIT sync' button on the wimax-enable.php page is pressed. All GENI members that have ever been a member of SOME wireless-enabled project -are made members of the 'holdingpen' group on ORBIT. In addition, they +are made members of the `holdingpen' group on ORBIT. In addition, they are made members of any group corresponding to any wireless-enabled projects to which they belong. When these projects are no longer wireless-enabled, or when the given user is not a member of such a project, their membership @@ -32,23 +34,34 @@ holdingpen group remains. .SH OPTIONS .TP -\fB--holdingpen_group -Name of group holding all GENI users regardless of additional membership in wireless-enabled group. Default: geni-HOLDINGPEN. +\fB--holdingpen_group +Name of group holding all GENI users regardless of additional membership in +wireless-enabled group. Default: geni-HOLDINGPEN. .TP -\fB--holdingpen_admin +\fB--holdingpen_admin Username in GENI CH of admin of holdingpen group. Default: agosain. .TP -\fB--project -[Optional]Name of project for which to perform sync between GENI and ORBIT state. +\fB--project +[Optional]Name of project for which to perform sync between GENI and ORBIT +state. .TP -\fB--user +\fB--user [Optional]Name of user for which to perform sync between GENI and ORBIT state. .TP \fB--cleanup -Remove all ORBIT groups and members that don't correspond to current GENI wireless-enabled projects and members. Default: false. +Remove all ORBIT groups and members that don't correspond to current GENI +wireless-enabled projects and members. Default: false. .TP -\fB--settings_file -Name of settings file from which to parse system-local information. Default: /etc/geni-ch/settings.php. - +\fB-k, --key +User/service key for making API calls. Must be an authority or operator. +.TP +\fB-c, --certificate +User/service certificate for making API calls. Must be an authority or +operator. +.TP +\fB-u, --url +URL of service registry. Member Authority, Slice Authority, and WiMAX (Orbit) +service are looked up in the service registry. + .SH AUTHOR geni-sync-wireless was written by Raytheon BBN Technologies. From d8ed766653a5b36be66c5cbd602d46490a6f8bbb Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 13:57:42 -0400 Subject: [PATCH 09/10] Delete outdated TODOs in sync wireless --- bin/geni-sync-wireless | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bin/geni-sync-wireless b/bin/geni-sync-wireless index e1ad1fec..300011f8 100644 --- a/bin/geni-sync-wireless +++ b/bin/geni-sync-wireless @@ -655,11 +655,6 @@ class WirelessProjectManager: for p in all_projects.values() if self.is_wimax_project(xmlrpc_client, p)} # Now add users to the wimax_projects - # TODO: if --user was specified, only include that username - # Do this by checking if the user's urn .endswith(username) - # if self._username and not urn.endswith(self.__username): - # contine - for proj in wimax_projects.values(): proj['members'] = self.get_project_members(xmlrpc_client, proj, user_name) @@ -732,7 +727,6 @@ class WirelessProjectManager: # TODO: Why do the ssh keys have a '\n' at the end? # Can ''.rstrip('\n') be used to *safely* strip it away? members[k]['ssh_keys'].append(ssh_key['KEY_PUBLIC']) - # TODO: Map response keys to user_info keys (see above) all_urns = members.keys() for urn in all_urns: for ma_key,new_key in GENI.MA.FieldMap.iteritems(): From 72a1db1be479d4c4b4ee0153462712c8c7bcc622 Mon Sep 17 00:00:00 2001 From: Tom Mitchell Date: Tue, 13 Sep 2016 13:58:10 -0400 Subject: [PATCH 10/10] Note completion of 1733 in CHANGES --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 31539958..35b90001 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,8 @@ ([#1731](https://github.com/GENI-NSF/geni-portal/issues/1731)) * Change /usr/local to /usr in doc to match RPM installation ([#1732](https://github.com/GENI-NSF/geni-portal/issues/1732)) +* Use clearinghouse APIs for geni-sync-wireless + ([#1733](https://github.com/GENI-NSF/geni-portal/issues/1733)) * geni-fetch-amdown fails outside GPO Lab ([#1734](https://github.com/GENI-NSF/geni-portal/issues/1734)) * Streamline and unify edit membership and edit project paths