From 2701368779721ab5cdc9ae054cc355c95439e72c Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Sat, 10 Aug 2013 19:15:42 -0700 Subject: [PATCH 1/6] Untested, but general idea of add_key thread --- modules/evekeys.py | 134 +++++++++++++++++++++++++++++++++++++++++++++ schema.py | 20 +++++++ 2 files changed, 154 insertions(+) create mode 100644 modules/evekeys.py diff --git a/modules/evekeys.py b/modules/evekeys.py new file mode 100644 index 0000000..99d10c3 --- /dev/null +++ b/modules/evekeys.py @@ -0,0 +1,134 @@ +from Queue import queue + +import logging +import threading +import time + +import evelink +import evelink.cache.sqlite + +from kitnirc.modular import Module + +import schema + + +_log = logging.getLogger(__name__) + + +class EveKeyError(Exception): + pass + + +class EveKeysModule(Module): + """Module to look up details on an EVE API key.""" + + def __init__(self, *args, **kwargs): + super(EveKeysModule, self).__init__(*args, **kwargs) + + self.cache = evelink.cache.sqlite.SqliteCache(".evecache.sqlite3") + self.results = queue() + self.tasks = queue() + + for i in range(5): + t = threading.Thread(target=self._worker) + t.daemon = True + t.start() + + self._stop = False + + def start(self, *args, **kwargs): + super(EveKeysModule, self).start(*args, **kwargs) + self._stop = False + + def stop(self, *args, **kwargs): + super(EveKeysModule, self).stop(*args, **kwargs) + self.stop = True + for _ in range(10): + if self.tasks.empty(): + break + + _log.info("Evekeys still has %d threads outstanding." % + self.tasks.qsize()) + time.sleep(1) + + if not self.tasks.empty(): + _log.warning("Evekeys giving up with %d threads outstanding." % + self.tasks.qsize()) + + def _worker(): + while not self.stop: + request = self.tasks.get() + _add_key(request) + self.tasks.task_done() + + def _add_key(request): + keyid = request['keyid'] + vcode = request['vcode'] + irc_account = request['metadata']['account'].lower() + + try: + api = evelink.api.API(api_key(keyid, vcode), cache=self.cache) + account = evelink.account.Account(api=api) + result = account.key_info() + except evelink.APIError as e: + _log.warn("Error loading API key(%s): %s" % (keyid, e)) + self.results.put((request, "Failed to load api key.")) + return + + if result: + _log.debug("key: %s, characters: %s" % (keyid, + ", ".join(char['name'] for char + in result['characters'].itervalues()))) + else: + _log.warn("key: %s, invalid key.") + self.results.put((request, "Invalid key.")) + return + + try: + summary = _save_key_info(keyid, + vcode, + irc_account, + result['characters']) + self.results.put((request, summary)) + return + + except DatabaseError as e: + _log.warn("Database error saving key(%s): %s" % (keyid, e)) + self.results.put((request, "Database error, try again later.")) + return + + def _save_key_info(keyid, vcode, irc_account, characters): + session = schema.Session() + + irc_account = metadata['account'].lower() + account, _ = schema.find_or_create(session, + schema.Account, + account=irc_account) + + # add key to the account + key, _ = schema.find_or_create(session, + schema.ApiKey, + keyid=keyid, + vcode=vcode) + + # add characters + for character in characters.itervalues(): + data = { 'name': character['name'], + 'corp': character['corp']['name'] } + char, _ = schema.find_or_create(session, + schema.Character, + data=data, + name=character['name']) + + session.commit() + return "%d characters added: %s" % ( + len(characters), + ", ".join([char['name'] for char in characters])) + + + def add_key(self, **kwargs): + self.tasks.put(kwargs) + + + +# vim: set ts=4 sts=4 sw=4 et: diff --git a/schema.py b/schema.py index 54980f2..d6f4c5b 100644 --- a/schema.py +++ b/schema.py @@ -11,6 +11,26 @@ def init_db(path): Base.metadata.create_all(engine) Session.configure(bind=engine) +def find_or_create(session, model, data=None, **kwargs): + """Find or create an object. + + returns: + (possibly new) objects, + was it created + """ + + instance = session.query(model).filter_by(**kwargs).first() + + if instance: + return instance, False + else: + if data: + instance = model(**data) + else: + instance = model(**kwargs) + session.add(instance) + return instance, True + account_api_m2m = Table('account_key_m2m', Base.metadata, Column('account_pk', String, ForeignKey('account.account')), Column('api_key_pk', Integer, ForeignKey('api_key.keyid')) From 2377fd5a7b3e308ec67799cf80d153ae63b9699a Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Mon, 12 Aug 2013 18:52:12 -0700 Subject: [PATCH 2/6] Don't block forever so that we keep checking the stop flag. --- modules/evekeys.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/modules/evekeys.py b/modules/evekeys.py index 99d10c3..1e3871e 100644 --- a/modules/evekeys.py +++ b/modules/evekeys.py @@ -1,4 +1,4 @@ -from Queue import queue +from Queue import queue, Empty import logging import threading @@ -57,7 +57,11 @@ def stop(self, *args, **kwargs): def _worker(): while not self.stop: - request = self.tasks.get() + try: + request = self.tasks.get(True, 5) + except Empty: + continue + _add_key(request) self.tasks.task_done() @@ -125,10 +129,15 @@ def _save_key_info(keyid, vcode, irc_account, characters): len(characters), ", ".join([char['name'] for char in characters])) - def add_key(self, **kwargs): - self.tasks.put(kwargs) + """Look up a given API key, associate with account and characters. + Args: + keyid - API key id. + vcode - API key verification. + metadata['account'] - IRC account name. + """ + self.tasks.put(kwargs) # vim: set ts=4 sts=4 sw=4 et: From cde7cebcd19d3a8840d3f75ef9886c4ff64fbeb5 Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Tue, 13 Aug 2013 14:21:23 -0700 Subject: [PATCH 3/6] Queue.Queue, not Queue.queue, silly me. --- modules/evekeys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/evekeys.py b/modules/evekeys.py index 1e3871e..e44bcd6 100644 --- a/modules/evekeys.py +++ b/modules/evekeys.py @@ -1,4 +1,4 @@ -from Queue import queue, Empty +from Queue import Queue, Empty import logging import threading @@ -26,8 +26,8 @@ def __init__(self, *args, **kwargs): super(EveKeysModule, self).__init__(*args, **kwargs) self.cache = evelink.cache.sqlite.SqliteCache(".evecache.sqlite3") - self.results = queue() - self.tasks = queue() + self.results = Queue() + self.tasks = Queue() for i in range(5): t = threading.Thread(target=self._worker) From b3b9d85401204aecee6083020530666a18dcedd9 Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Tue, 13 Aug 2013 14:32:30 -0700 Subject: [PATCH 4/6] Address review comments: import order and type; log messages --- modules/evekeys.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/modules/evekeys.py b/modules/evekeys.py index e44bcd6..fddc310 100644 --- a/modules/evekeys.py +++ b/modules/evekeys.py @@ -1,12 +1,10 @@ -from Queue import Queue, Empty - import logging +import Queue import threading import time import evelink import evelink.cache.sqlite - from kitnirc.modular import Module import schema @@ -15,10 +13,6 @@ _log = logging.getLogger(__name__) -class EveKeyError(Exception): - pass - - class EveKeysModule(Module): """Module to look up details on an EVE API key.""" @@ -26,8 +20,8 @@ def __init__(self, *args, **kwargs): super(EveKeysModule, self).__init__(*args, **kwargs) self.cache = evelink.cache.sqlite.SqliteCache(".evecache.sqlite3") - self.results = Queue() - self.tasks = Queue() + self.results = Queue.Queue() + self.tasks = Queue.Queue() for i in range(5): t = threading.Thread(target=self._worker) @@ -47,19 +41,21 @@ def stop(self, *args, **kwargs): if self.tasks.empty(): break - _log.info("Evekeys still has %d threads outstanding." % - self.tasks.qsize()) + _log.info( + "Shutting down - EveKeys still has %d threads outstanding." % + self.tasks.qsize()) time.sleep(1) if not self.tasks.empty(): - _log.warning("Evekeys giving up with %d threads outstanding." % - self.tasks.qsize()) + _log.warning( + "Shutting down - giving up on EveKeys %d outstanding threads." % + self.tasks.qsize()) def _worker(): while not self.stop: try: request = self.tasks.get(True, 5) - except Empty: + except Queue.Empty: continue _add_key(request) From ae59381b305184569d0bac265807d73134961186 Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Tue, 13 Aug 2013 17:45:57 -0700 Subject: [PATCH 5/6] Remove undeeded find_or_create, rewrite key saving code to actually work. --- modules/evekeys.py | 63 ++++++++++++++++++++++++++++------------------ schema.py | 26 +++---------------- 2 files changed, 42 insertions(+), 47 deletions(-) diff --git a/modules/evekeys.py b/modules/evekeys.py index fddc310..50678b2 100644 --- a/modules/evekeys.py +++ b/modules/evekeys.py @@ -42,19 +42,19 @@ def stop(self, *args, **kwargs): break _log.info( - "Shutting down - EveKeys still has %d threads outstanding." % + "EveKeys shutting down - %d threads still outstanding." % self.tasks.qsize()) time.sleep(1) if not self.tasks.empty(): _log.warning( - "Shutting down - giving up on EveKeys %d outstanding threads." % + "EveKeys shutting down - giving up on %d outstanding threads." % self.tasks.qsize()) def _worker(): while not self.stop: try: - request = self.tasks.get(True, 5) + request = self.tasks.get(True, 1) except Queue.Empty: continue @@ -64,7 +64,7 @@ def _worker(): def _add_key(request): keyid = request['keyid'] vcode = request['vcode'] - irc_account = request['metadata']['account'].lower() + irc_account = request['metadata']['account'] try: api = evelink.api.API(api_key(keyid, vcode), cache=self.cache) @@ -72,7 +72,7 @@ def _add_key(request): result = account.key_info() except evelink.APIError as e: _log.warn("Error loading API key(%s): %s" % (keyid, e)) - self.results.put((request, "Failed to load api key.")) + self.results.put((request, "Failed to load api key %s." % keyid)) return if result: @@ -89,43 +89,53 @@ def _add_key(request): vcode, irc_account, result['characters']) - self.results.put((request, summary)) return - except DatabaseError as e: _log.warn("Database error saving key(%s): %s" % (keyid, e)) self.results.put((request, "Database error, try again later.")) return + self.results.put((request, summary)) + def _save_key_info(keyid, vcode, irc_account, characters): session = schema.Session() - irc_account = metadata['account'].lower() - account, _ = schema.find_or_create(session, - schema.Account, - account=irc_account) + irc_account = metadata['account'] - # add key to the account - key, _ = schema.find_or_create(session, - schema.ApiKey, - keyid=keyid, - vcode=vcode) + # find or create account + account = session.query(schema.Account).get(irc_account) + if not account: + account = schema.Account(irc_account, False) - # add characters + # find or create an associated key + key = session.query(schema.ApiKey).get(keyid) + if key: + key.vcode = vcode + else: + key = schema.ApiKey(keyid, vcode) + account.keys.add(key) + + # update/delete existing characters + for character in key.characters: + if character.name in characters: + character.corp = characters[character.name]['corp'] + del(characters[character.name]) + else: + session.delete(character) + + # add new characters for character in characters.itervalues(): - data = { 'name': character['name'], - 'corp': character['corp']['name'] } - char, _ = schema.find_or_create(session, - schema.Character, - data=data, - name=character['name']) + key.characters.add(schema.Character(character['name'], + character['corp'])) + # save everything + session.add(account) session.commit() return "%d characters added: %s" % ( len(characters), ", ".join([char['name'] for char in characters])) - def add_key(self, **kwargs): + def add_key(self, keyid, vcode, metadata): """Look up a given API key, associate with account and characters. Args: @@ -133,7 +143,10 @@ def add_key(self, **kwargs): vcode - API key verification. metadata['account'] - IRC account name. """ - self.tasks.put(kwargs) + request = { 'keyid': keyid, + 'vcode': vcode, + 'metadata': metadata } + self.tasks.put(request) # vim: set ts=4 sts=4 sw=4 et: diff --git a/schema.py b/schema.py index d6f4c5b..a43e6dc 100644 --- a/schema.py +++ b/schema.py @@ -11,26 +11,6 @@ def init_db(path): Base.metadata.create_all(engine) Session.configure(bind=engine) -def find_or_create(session, model, data=None, **kwargs): - """Find or create an object. - - returns: - (possibly new) objects, - was it created - """ - - instance = session.query(model).filter_by(**kwargs).first() - - if instance: - return instance, False - else: - if data: - instance = model(**data) - else: - instance = model(**kwargs) - session.add(instance) - return instance, True - account_api_m2m = Table('account_key_m2m', Base.metadata, Column('account_pk', String, ForeignKey('account.account')), Column('api_key_pk', Integer, ForeignKey('api_key.keyid')) @@ -50,7 +30,8 @@ class Account(Base): keys = relationship("ApiKey", backref="accounts", - secondary=account_api_m2m) + secondary=account_api_m2m, + collection_class=set) def __init__(self, account, is_admin=False): self.account = account @@ -69,7 +50,8 @@ class ApiKey(Base): characters = relationship("Character", backref="api", - secondary=api_character_m2m) + secondary=api_character_m2m, + collection_class=set) def __init__(self, key_id, vcode, last_check=None): self.keyid = key_id From e5242dc4f44bcda303d06a59270461f11a837eaa Mon Sep 17 00:00:00 2001 From: Dan Boger Date: Wed, 14 Aug 2013 13:10:24 -0700 Subject: [PATCH 6/6] Explicitly get and pass the account value, making metadata a purly passthrough object --- modules/evekeys.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/evekeys.py b/modules/evekeys.py index 50678b2..d2a15e6 100644 --- a/modules/evekeys.py +++ b/modules/evekeys.py @@ -64,7 +64,7 @@ def _worker(): def _add_key(request): keyid = request['keyid'] vcode = request['vcode'] - irc_account = request['metadata']['account'] + irc_account = request['account'] try: api = evelink.api.API(api_key(keyid, vcode), cache=self.cache) @@ -100,8 +100,6 @@ def _add_key(request): def _save_key_info(keyid, vcode, irc_account, characters): session = schema.Session() - irc_account = metadata['account'] - # find or create account account = session.query(schema.Account).get(irc_account) if not account: @@ -135,16 +133,18 @@ def _save_key_info(keyid, vcode, irc_account, characters): len(characters), ", ".join([char['name'] for char in characters])) - def add_key(self, keyid, vcode, metadata): + def add_key(self, keyid, vcode, account, metadata): """Look up a given API key, associate with account and characters. Args: keyid - API key id. vcode - API key verification. - metadata['account'] - IRC account name. + account - IRC account name. + metadata - context object passed through """ request = { 'keyid': keyid, 'vcode': vcode, + 'account': account, 'metadata': metadata } self.tasks.put(request)