From 3030d8871a3cd6fe4c324655822e72619ae29a68 Mon Sep 17 00:00:00 2001 From: tsungwu Date: Thu, 27 Apr 2017 16:18:45 +0800 Subject: [PATCH 1/2] [v2.1.3] add ESDAO module --- webapp/app/__init__.py | 2 +- webapp/app/api/__init__.py | 4 +- webapp/app/api/es/__init__.py | 0 webapp/app/api/es/apiController.py | 91 ++++++++++++++++++++++++++++++ webapp/app/config.Development.cfg | 5 +- webapp/app/dao/__init__.py | 7 ++- webapp/app/dao/esDAO.py | 53 +++++++++++++++++ 7 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 webapp/app/api/es/__init__.py create mode 100644 webapp/app/api/es/apiController.py create mode 100644 webapp/app/dao/esDAO.py diff --git a/webapp/app/__init__.py b/webapp/app/__init__.py index bbb5897..1ac958e 100644 --- a/webapp/app/__init__.py +++ b/webapp/app/__init__.py @@ -45,7 +45,7 @@ def create_app(flask_config_name=None): from app.model.userModel import User - me = User('admin', 'admin@example.com',None, None) + me = User('admin', 'admin@example.com', None, None) daoPool.sqlDAO.session.add(me) daoPool.sqlDAO.session.commit() diff --git a/webapp/app/api/__init__.py b/webapp/app/api/__init__.py index 5f37ce3..3ed9447 100644 --- a/webapp/app/api/__init__.py +++ b/webapp/app/api/__init__.py @@ -3,9 +3,10 @@ from app.api.cve.apiController import api as ns1 from app.api.user.apiController import api as ns2 from app.api.oauth.apiController import api as ns3 +from app.api.es.apiController import api as ns4 -api = Api(version='2.1', \ +api = Api(version='2.1.3', \ title='Flask Restful plus Api', \ doc='/api', \ description='Document for Restful api', \ @@ -16,3 +17,4 @@ api.add_namespace(ns1, path='/api/cves') api.add_namespace(ns2, path='/api/users') api.add_namespace(ns3, path='/api/oauth') +api.add_namespace(ns4, path='/api/es') diff --git a/webapp/app/api/es/__init__.py b/webapp/app/api/es/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/webapp/app/api/es/apiController.py b/webapp/app/api/es/apiController.py new file mode 100644 index 0000000..7e98141 --- /dev/null +++ b/webapp/app/api/es/apiController.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +import json, re +import os + +from flask_restplus import reqparse, Resource, Api, fields, Namespace, abort +from flask import request, session, make_response, redirect, Response +from elasticsearch_dsl import Q, A, Search + +from app.util.regToolBox import RegToolBox +from app.util.exceptionHandler import ExcepitonHandler +from app.dao import daoPool + + +api = Namespace('es', description='es operation') +esDAO = daoPool.esDAO + +regBox = RegToolBox() +excpHandler = ExcepitonHandler() + + +## Request parsing +parser = reqparse.RequestParser() +parser.add_argument('from', type=int, required=True, help='Offset from the first result') +parser.add_argument('size', type=int, required=True, help='Amount of cve item to be returned') + +parserAgg = reqparse.RequestParser() +parserAgg.add_argument('size', type=int, required=True, help='Number of item to be returned') + +@api.route('/search') +class GoogleLogin(Resource): + """google resource""" + def __init__(self, Resource): + self.api = api + esDAO.setIndexAndType("cve", "detail") + + + @api.expect(parser) + @api.doc(description='ESDAO Search example') + def get(self): + """ Return es result """ + args = parser.parse_args() + + size = args['size'] + fromOffset = args['from'] + + q = Q() + sortQ = {} + + result = {} + resultList = [] + resp = esDAO.search(q, sortQ, fromOffset, size) + + for hit in resp.to_dict()['hits']['hits']: + resultList.append(hit) + + result['result'] = resultList + + return result + +@api.route('/aggs') +class EsAggs(Resource): + """ """ + def __init__(self, Resource): + self.api = api + esDAO.setIndexAndType("cve", "detail") + + + @api.expect(parserAgg) + @api.doc(description='ESDAO Aggregation example' \ + 'Get top `size` recent item') + def get(self): + """ get top recent result """ + args = parserAgg.parse_args() + + topSize = args['size'] + + q = Q() + s = Search().query(q) + sortDate = [{"original_release_date": {"order": "desc"}}] + a_date = A('top_hits', sort=sortDate, size=topSize) + s.aggs.metric('_topDate', a_date) + + result = {} + resultList = [] + resp = esDAO.aggs(s).to_dict() + for hit in resp['_topDate']['hits']['hits']: + resultList.append(hit) + + result['_topDate'] = resultList + + return result diff --git a/webapp/app/config.Development.cfg b/webapp/app/config.Development.cfg index 87905da..3e68c2c 100644 --- a/webapp/app/config.Development.cfg +++ b/webapp/app/config.Development.cfg @@ -8,8 +8,9 @@ JSON_AS_ASCII = 'false' SECRET_KEY = 'f5557d4fcf727a981a3c315aca733eefa2996f7c7cdae1fa7e0de28522820bb0' ## Elasticsearch settings -#ELASTICSEARCH_HOST = '' -#ELASTICSEARCH_PORT = '9200' +ELASTICSEARCH_HOST = 'http://192.168.30.49' +ELASTICSEARCH_PORT = '9200' + ## CORS settings CORS_ORIGIN = '*' diff --git a/webapp/app/dao/__init__.py b/webapp/app/dao/__init__.py index 5d0ac38..6daafd8 100644 --- a/webapp/app/dao/__init__.py +++ b/webapp/app/dao/__init__.py @@ -1,9 +1,11 @@ from flask_sqlalchemy import SQLAlchemy from app.model import init_model - +from app.dao.esDAO import ElasticsearchDAO class DaoPool(): + sqlDAO = None + esDAO = None def __init__(self): pass @@ -13,4 +15,7 @@ def init_app(self, app): self.sqlDAO = SQLAlchemy(app) init_model(self.sqlDAO) + self.esDAO = ElasticsearchDAO(app.config['ELASTICSEARCH_HOST'], \ + app.config['ELASTICSEARCH_PORT']) + daoPool = DaoPool() \ No newline at end of file diff --git a/webapp/app/dao/esDAO.py b/webapp/app/dao/esDAO.py new file mode 100644 index 0000000..c54efd9 --- /dev/null +++ b/webapp/app/dao/esDAO.py @@ -0,0 +1,53 @@ +import sys +import datetime +import logging +from elasticsearch import Elasticsearch +from flask_restplus import reqparse, Resource +from elasticsearch_dsl import Search, Q, A + + +reload(sys) +sys.setdefaultencoding('utf-8') +logger = logging.getLogger('ESDAO') +class ElasticsearchDAO(): + client = None + indexES = None + typeES = None + esearch = None + def __init__(self, host, port): + url = "%s:%s" % (host, port) + try: + self.client = Elasticsearch([url], send_get_body_as="POST") + except: + logger.error('elasticsearch cannot connect') + + def setIndexAndType(self, index, type): + self.indexES = index + self.typeES = type + self.esearch = Search(using=self.client, index=self.indexES, doc_type=self.typeES) + + + def saveJson(self, json): + """put json to ES """ + res = self.client.index(index=self.indexES, doc_type=self.typeES, body=json) + return res + + + def aggs(self, searchQuery): + s = self.esearch.query() + s.aggs = searchQuery.aggs + s.query = searchQuery.query + s = s[0:0] + response = s.execute() + + return response.aggregations + + def search(self, q, sortQ, fromOffset, size): + s = self.esearch.query(q) \ + .sort({"_score":{"order":"desc"}}, sortQ) + s = s[fromOffset:fromOffset+size] + response = s.execute() + + return response + + From d86ff8ec8512c4c5ea0fe15e1d183f81388d4615 Mon Sep 17 00:00:00 2001 From: tsungwu Date: Thu, 27 Apr 2017 16:35:46 +0800 Subject: [PATCH 2/2] [v2.2] update readme --- README.md | 25 ++++++++++++++++++++----- webapp/app/api/__init__.py | 2 +- webapp/app/api/es/apiController.py | 2 +- webapp/app/config.Development.cfg | 2 +- webapp/app/config.Production.cfg | 4 ++-- webapp/app/config.Testing.cfg | 4 ++-- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3bde722..913c2f5 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,16 @@ Using Flask to build a Restful API Server with Swagger document. Integration with Flask-restplus, Flask-Cors, Flask-Testing, Flask-SQLalchemy,and Flask-OAuth extensions. ### Extension: -- Restful: Flask-restplus +- Restful: [Flask-restplus](http://flask-restplus.readthedocs.io/en/stable/) -- SQL ORM: Flask-SQLalchemy +- SQL ORM: [Flask-SQLalchemy](http://flask-sqlalchemy.pocoo.org/2.1/) -- Testing: Flask-Testing +- Testing: [Flask-Testing](http://flask.pocoo.org/docs/0.12/testing/) + +- OAuth: [Flask-OAuth](https://pythonhosted.org/Flask-OAuth/) + +- ESDAO: [elasticsearch](https://elasticsearch-py.readthedocs.io/en/master/) , [elasticsearch-dsl](http://elasticsearch-dsl.readthedocs.io/en/latest/index.html) -- OAuth: Flask-OAuth ## Installation @@ -86,9 +89,15 @@ JSON_SORT_KEYS : By default Flask will serialize JSON objects in a way that the - [reference¶](http://flask.pocoo.org/docs/0.12/config/) -#### OAuth setup +### OAuth Setup add your `client_id` and `client_secret` into config file. +### ESDAO Setup +add your `ES host` and `ES port` into config file + + + + ## Run Flask ### Run flask for develop ``` @@ -128,14 +137,20 @@ Offical Website - [Flask restplus](http://flask-restplus.readthedocs.io/en/stable/) - [Flask-SQLalchemy](http://flask-sqlalchemy.pocoo.org/2.1/) - [Flask-OAuth](https://pythonhosted.org/Flask-OAuth/) +- [elasticsearch-dsl](http://elasticsearch-dsl.readthedocs.io/en/latest/index.html) +- [gunicorn](http://gunicorn.org/) Tutorial - [Flask Overview](https://www.slideshare.net/maxcnunes1/flask-python-16299282) - [In Flask we trust](http://igordavydenko.com/talks/ua-pycon-2012.pdf) +[Wiki Page](https://github.com/tsungtwu/flask-example/wiki) + + ## Changelog +- Version 2.2 : add ESDAO module - Version 2.1 : add OAuth extension: FLASK-OAuth, and google oauth example - Version 2.0 : add SQL ORM extension: FLASK-SQLAlchemy - Version 1.1 : update nosetest diff --git a/webapp/app/api/__init__.py b/webapp/app/api/__init__.py index 3ed9447..c4c0079 100644 --- a/webapp/app/api/__init__.py +++ b/webapp/app/api/__init__.py @@ -6,7 +6,7 @@ from app.api.es.apiController import api as ns4 -api = Api(version='2.1.3', \ +api = Api(version='2.2', \ title='Flask Restful plus Api', \ doc='/api', \ description='Document for Restful api', \ diff --git a/webapp/app/api/es/apiController.py b/webapp/app/api/es/apiController.py index 7e98141..fa092b9 100644 --- a/webapp/app/api/es/apiController.py +++ b/webapp/app/api/es/apiController.py @@ -11,7 +11,7 @@ from app.dao import daoPool -api = Namespace('es', description='es operation') +api = Namespace('es', description='Es operation') esDAO = daoPool.esDAO regBox = RegToolBox() diff --git a/webapp/app/config.Development.cfg b/webapp/app/config.Development.cfg index 3e68c2c..eb76e45 100644 --- a/webapp/app/config.Development.cfg +++ b/webapp/app/config.Development.cfg @@ -8,7 +8,7 @@ JSON_AS_ASCII = 'false' SECRET_KEY = 'f5557d4fcf727a981a3c315aca733eefa2996f7c7cdae1fa7e0de28522820bb0' ## Elasticsearch settings -ELASTICSEARCH_HOST = 'http://192.168.30.49' +ELASTICSEARCH_HOST = '' ELASTICSEARCH_PORT = '9200' diff --git a/webapp/app/config.Production.cfg b/webapp/app/config.Production.cfg index a3498bc..5945ee6 100644 --- a/webapp/app/config.Production.cfg +++ b/webapp/app/config.Production.cfg @@ -6,8 +6,8 @@ TESTING = False JSON_AS_ASCII = 'false' #elasticsearch settings -#ELASTICSEARCH_HOST = '' -#ELASTICSEARCH_PORT = '9200' +ELASTICSEARCH_HOST = '' +ELASTICSEARCH_PORT = '9200' #CORS settings CORS_ORIGIN = '*' diff --git a/webapp/app/config.Testing.cfg b/webapp/app/config.Testing.cfg index 9a8ef1f..cc4a135 100644 --- a/webapp/app/config.Testing.cfg +++ b/webapp/app/config.Testing.cfg @@ -6,8 +6,8 @@ TESTING = True JSON_AS_ASCII = 'false' #elasticsearch settings -#ELASTICSEARCH_HOST = '' -#ELASTICSEARCH_PORT = '9200' +ELASTICSEARCH_HOST = '' +ELASTICSEARCH_PORT = '9200'