Merge branch 'release/flask_example_v2.2'

README.md

@@ -5,13 +5,16 @@ Using Flask to build a Restful API Server with Swagger document.
 Integration with Flask-restplus, Flask-Cors, Flask-Testing, Flask-SQLalchemy,and Flask-OAuth extensions.
 
 ### Extension:
-- Restful: Flask-restplus
+- Restful: [Flask-restplus](http://flask-restplus.readthedocs.io/en/stable/)
 
-- SQL ORM: Flask-SQLalchemy 
+- SQL ORM: [Flask-SQLalchemy](http://flask-sqlalchemy.pocoo.org/2.1/)
 
-- Testing: Flask-Testing
+- Testing: [Flask-Testing](http://flask.pocoo.org/docs/0.12/testing/)
+
+- OAuth: [Flask-OAuth](https://pythonhosted.org/Flask-OAuth/)
+
+- ESDAO: [elasticsearch](https://elasticsearch-py.readthedocs.io/en/master/) , [elasticsearch-dsl](http://elasticsearch-dsl.readthedocs.io/en/latest/index.html)
 
-- OAuth: Flask-OAuth
 
 ## Installation
 
@@ -86,9 +89,15 @@ JSON_SORT_KEYS : By default Flask will serialize JSON objects in a way that the
 - [reference¶](http://flask.pocoo.org/docs/0.12/config/)
 
 
-#### OAuth setup
+### OAuth Setup
 add your `client_id` and `client_secret` into config file.
 
+### ESDAO Setup
+add your `ES host` and `ES port` into config file 
+
+
+
+
 ## Run Flask
 ### Run flask for develop
 ```
@@ -128,14 +137,20 @@ Offical Website
 - [Flask restplus](http://flask-restplus.readthedocs.io/en/stable/)
 - [Flask-SQLalchemy](http://flask-sqlalchemy.pocoo.org/2.1/)
 - [Flask-OAuth](https://pythonhosted.org/Flask-OAuth/)
+- [elasticsearch-dsl](http://elasticsearch-dsl.readthedocs.io/en/latest/index.html)
+- [gunicorn](http://gunicorn.org/)
 
 Tutorial
 
 - [Flask Overview](https://www.slideshare.net/maxcnunes1/flask-python-16299282)
 - [In Flask we trust](http://igordavydenko.com/talks/ua-pycon-2012.pdf)
 
+[Wiki Page](https://github.com/tsungtwu/flask-example/wiki)
+
+
 
 ## Changelog
+- Version 2.2 : add ESDAO module
 - Version 2.1 : add OAuth extension: FLASK-OAuth, and google oauth example
 - Version 2.0 : add SQL ORM extension: FLASK-SQLAlchemy
 - Version 1.1 : update nosetest

webapp/app/__init__.py

@@ -45,7 +45,7 @@ def create_app(flask_config_name=None):
 
     from app.model.userModel import User
 
-    me = User('admin', '[email protected]',None, None)
+    me = User('admin', '[email protected]', None, None)
     daoPool.sqlDAO.session.add(me)
     daoPool.sqlDAO.session.commit()
 

webapp/app/api/__init__.py

@@ -3,9 +3,10 @@
 from app.api.cve.apiController import api as ns1
 from app.api.user.apiController import api as ns2
 from app.api.oauth.apiController import api as ns3
+from app.api.es.apiController import api as ns4
 
 
-api = Api(version='2.1', \
+api = Api(version='2.2', \
             title='Flask Restful plus Api', \
             doc='/api', \
             description='Document for Restful api', \
@@ -16,3 +17,4 @@
 api.add_namespace(ns1, path='/api/cves')
 api.add_namespace(ns2, path='/api/users')
 api.add_namespace(ns3, path='/api/oauth')
+api.add_namespace(ns4, path='/api/es')

webapp/app/api/es/apiController.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import json, re
+import os
+
+from flask_restplus import reqparse, Resource, Api, fields, Namespace, abort
+from flask import request, session, make_response, redirect, Response
+from elasticsearch_dsl import Q, A, Search
+
+from app.util.regToolBox import RegToolBox
+from app.util.exceptionHandler import ExcepitonHandler
+from app.dao import daoPool
+
+
+api = Namespace('es', description='Es operation')
+esDAO = daoPool.esDAO
+
+regBox =  RegToolBox()
+excpHandler = ExcepitonHandler()
+
+
+## Request parsing
+parser = reqparse.RequestParser()
+parser.add_argument('from', type=int, required=True, help='Offset from the first result')
+parser.add_argument('size', type=int, required=True, help='Amount of cve item to be returned')
+
+parserAgg = reqparse.RequestParser()
+parserAgg.add_argument('size', type=int, required=True, help='Number of item to be returned')
+
+@api.route('/search')
+class GoogleLogin(Resource):
+    """google resource"""
+    def __init__(self, Resource):
+        self.api = api
+        esDAO.setIndexAndType("cve", "detail")
+
+
+    @api.expect(parser)
+    @api.doc(description='ESDAO Search example')    
+    def get(self):
+        """ Return es result """
+        args = parser.parse_args()
+
+        size = args['size']
+        fromOffset = args['from']
+
+        q = Q()
+        sortQ = {}
+
+        result = {}
+        resultList = []
+        resp = esDAO.search(q, sortQ, fromOffset, size)
+
+        for hit in resp.to_dict()['hits']['hits']:
+            resultList.append(hit)
+
+        result['result'] = resultList
+
+        return result
+
+@api.route('/aggs')
+class EsAggs(Resource):
+    """ """
+    def __init__(self, Resource):
+        self.api = api
+        esDAO.setIndexAndType("cve", "detail")
+
+
+    @api.expect(parserAgg)
+    @api.doc(description='ESDAO Aggregation example' \
+                            'Get top `size` recent item')
+    def get(self):
+        """ get top  recent result """
+        args = parserAgg.parse_args()
+
+        topSize = args['size']
+
+        q = Q()
+        s = Search().query(q)
+        sortDate = [{"original_release_date": {"order": "desc"}}]
+        a_date = A('top_hits', sort=sortDate, size=topSize)
+        s.aggs.metric('_topDate', a_date)
+
+        result = {}
+        resultList = []
+        resp = esDAO.aggs(s).to_dict()
+        for hit in resp['_topDate']['hits']['hits']:
+            resultList.append(hit)
+
+        result['_topDate'] = resultList
+
+        return result

webapp/app/config.Development.cfg

@@ -8,8 +8,9 @@ JSON_AS_ASCII = 'false'
 SECRET_KEY = 'f5557d4fcf727a981a3c315aca733eefa2996f7c7cdae1fa7e0de28522820bb0'
 
 ## Elasticsearch settings
-#ELASTICSEARCH_HOST = ''
-#ELASTICSEARCH_PORT = '9200'
+ELASTICSEARCH_HOST = ''
+ELASTICSEARCH_PORT = '9200'
+
 
 ## CORS settings
 CORS_ORIGIN = '*'

webapp/app/config.Production.cfg

@@ -6,8 +6,8 @@ TESTING = False
 JSON_AS_ASCII = 'false'
 
 #elasticsearch settings
-#ELASTICSEARCH_HOST = ''
-#ELASTICSEARCH_PORT = '9200'
+ELASTICSEARCH_HOST = ''
+ELASTICSEARCH_PORT = '9200'
 
 #CORS settings
 CORS_ORIGIN = '*'

webapp/app/config.Testing.cfg

@@ -6,8 +6,8 @@ TESTING = True
 JSON_AS_ASCII = 'false'
 
 #elasticsearch settings
-#ELASTICSEARCH_HOST = ''
-#ELASTICSEARCH_PORT = '9200'
+ELASTICSEARCH_HOST = ''
+ELASTICSEARCH_PORT = '9200'
 
 
 

webapp/app/dao/__init__.py

@@ -1,9 +1,11 @@
 from flask_sqlalchemy import SQLAlchemy
 from app.model import init_model
-
+from app.dao.esDAO import ElasticsearchDAO
 
 class DaoPool():
+
     sqlDAO = None
+    esDAO = None
     def __init__(self):
         pass
 
@@ -13,4 +15,7 @@ def init_app(self, app):
         self.sqlDAO = SQLAlchemy(app)
         init_model(self.sqlDAO)
 
+        self.esDAO = ElasticsearchDAO(app.config['ELASTICSEARCH_HOST'], \
+                                        app.config['ELASTICSEARCH_PORT'])
+
 daoPool = DaoPool()

webapp/app/dao/esDAO.py

@@ -0,0 +1,53 @@
+import sys
+import datetime
+import logging
+from elasticsearch import Elasticsearch
+from flask_restplus import reqparse, Resource
+from elasticsearch_dsl import Search, Q, A
+
+
+reload(sys)
+sys.setdefaultencoding('utf-8')
+logger = logging.getLogger('ESDAO')
+class ElasticsearchDAO():
+    client = None
+    indexES = None
+    typeES = None
+    esearch = None
+    def __init__(self, host, port):
+        url = "%s:%s" % (host, port)
+        try:
+            self.client = Elasticsearch([url], send_get_body_as="POST")
+        except:
+            logger.error('elasticsearch cannot connect')
+
+    def setIndexAndType(self, index, type):
+        self.indexES = index
+        self.typeES = type
+        self.esearch = Search(using=self.client, index=self.indexES, doc_type=self.typeES)
+
+
+    def saveJson(self, json):
+        """put json to ES """
+        res = self.client.index(index=self.indexES, doc_type=self.typeES, body=json)
+        return res
+
+
+    def aggs(self, searchQuery):
+        s = self.esearch.query()
+        s.aggs = searchQuery.aggs
+        s.query = searchQuery.query
+        s = s[0:0]
+        response = s.execute()
+
+        return response.aggregations
+
+    def search(self, q, sortQ, fromOffset, size):
+        s = self.esearch.query(q) \
+            .sort({"_score":{"order":"desc"}}, sortQ)
+        s = s[fromOffset:fromOffset+size]
+        response = s.execute()
+
+        return response
+
+