Feature/send to pypln

capture/downloader.py

@@ -28,8 +28,8 @@
 from pymongo.errors import DuplicateKeyError
 import bson
 from dateutil.parser import parse
-
 import settings
+from nlp import send_to_pypln
 
 
 sys.path.append('/'.join(os.getcwd().split("/")[:-1]))

capture/load_into_pypln.py

@@ -1,4 +1,4 @@
-#-*- coding:utf-8 -*-
+# -*- coding:utf-8 -*-
 u"""
 Created on 03/04/14
 by fccoelho
@@ -19,39 +19,44 @@
 articles = client.MCDB.articles
 
 
-
-def load(corpus_name, skip, limit):
-    corpus = nlp.get_corpus(corpus_name)
-
+def load(skip, limit=0):
+    corpus = nlp.get_corpus()
+    articles_sent = 0
     filter_ = {'pypln_url': {'$exists': False}}
 
     find_kwargs = {'sort': [("_id", pymongo.DESCENDING)]}
     if skip:
         find_kwargs.update({'skip': skip})
     if limit:
         find_kwargs.update({'limit': limit})
-
-    cursor = articles.find(filter_, **find_kwargs)
-    for article in cursor:
-        pypln_document = nlp.send_to_pypln(article, corpus)
-        _id = article['_id']
-        articles.update({'_id': _id},
-                        {'$set': {"pypln_url": pypln_document.url}})
-        sys.stdout.write('inserted document with id {} into PyPLN\n'.format(_id))
-
-
-if __name__=="__main__":
+    if limit == 0:
+        count = articles.count()
+    else:
+        count = limit
+    while articles_sent < count:
+        cursor = articles.find(filter_, skip=articles_sent, limit=100, **find_kwargs)
+        for article in cursor:
+            pypln_document = nlp.send_to_pypln(article, corpus)
+            _id = article['_id']
+            articles.update({'_id': _id},
+                            {'$set': {"pypln_url": pypln_document.url}})
+            sys.stdout.write('inserted document {} of {}, with id {} into PyPLN\n'.format(articles_sent, count, _id))
+            articles_sent += 1
+
+
+if __name__ == "__main__":
     import argparse
+
     parser = argparse.ArgumentParser(description=("Load MediaCloud documents"
-        "into a PyPLN instance"))
+                                                  "into a PyPLN instance"))
 
-    parser.add_argument("-c", "--corpus_name", type=str, metavar="NAME",
-            default="MC_articles",
-            help="Uploads documents to a corpus named NAME")
+    # parser.add_argument("-c", "--corpus_name", type=str, metavar="NAME",
+    # default="MC_articles",
+    #         help="Uploads documents to a corpus named NAME")
     parser.add_argument("-l", "--limit", metavar='N', type=int, default=0,
-        help="Adds limit=N to the mongo query")
+                        help="Adds limit=N to the mongo query")
     parser.add_argument("-s", "--skip", metavar='N', type=int, default=0,
-        help="Adds skip=N to the mongo query")
+                        help="Adds skip=N to the mongo query")
     args = parser.parse_args()
 
-    load(args.corpus_name, args.skip, args.limit)
+    load(args.skip, args.limit)

capture/nlp.py

@@ -21,11 +21,12 @@
 
 ARTICLES.ensure_index([("pypln_url", pymongo.ASCENDING)], sparse=True)
 
-def get_corpus(corpus_name='MC_articles'):
+
+def get_corpus():
     """
     Return the existing Mediacloud corpus or create it and return.
-    :rtype : Corpus object
     """
+    corpus_name = settings.CORPUS_NAME
     try:
         article_corpus = pypln.add_corpus(name=corpus_name, description='MediaCloud Articles')
     except RuntimeError:

capture/settings.py

@@ -18,4 +18,4 @@
 ##########
 PYPLNHOST = "http://fgv.pypln.org/"
 PYPLN_CREDENTIALS = ("mediacloud2", "senha do mediacloud")
-
+CORPUS_NAME = "MC_articles"

capture/twitter/GeoLoc.py

@@ -9,7 +9,7 @@
 from dateutil import parser
 
 
-FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
+FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
 logging.basicConfig(filename='tweet_geoloc.log', format=FORMAT, level=logging.DEBUG)
 
 # Initialize connection