From 7914f5555975dcb6a9875cb02500119f8fe1e09e Mon Sep 17 00:00:00 2001 From: Modrzew Date: Sun, 24 Jul 2016 12:59:55 +0200 Subject: [PATCH 1/7] Simple report with basic stats about gathering session --- db.py | 91 +++++++++++++++++++++ templates/report.html | 182 ++++++++++++++++++++++++++++++++++++++++++ web.py | 78 +++++++++++++++++- 3 files changed, 350 insertions(+), 1 deletion(-) create mode 100644 templates/report.html diff --git a/db.py b/db.py index 41ca3ca04a..e62f54ebd3 100644 --- a/db.py +++ b/db.py @@ -1,3 +1,5 @@ +from datetime import datetime +import json import time from sqlalchemy.orm import sessionmaker @@ -7,6 +9,10 @@ from sqlalchemy.sql import not_ +with open('locales/pokemon.en.json') as f: + pokemon_names = json.load(f) + + try: import config DB_ENGINE = config.DB_ENGINE @@ -69,3 +75,88 @@ def get_sightings(session): if trash_list: query = query.filter(not_(Sighting.pokemon_id.in_(config.TRASH_IDS))) return query.all() + + +def get_session_stats(session): + min_max_query = session.execute(''' + SELECT + MIN(expire_timestamp) ts_min, + MAX(expire_timestamp) ts_max, + COUNT(*) + FROM `sightings`; + ''') + min_max_result = min_max_query.first() + per_hour_query = session.execute(''' + SELECT CAST(AVG(how_many) AS SIGNED) FROM ( + SELECT + CAST((expire_timestamp / 3600) AS SIGNED) ts_date, + COUNT(*) how_many + FROM `sightings` + GROUP BY ts_date ORDER BY ts_date + ) t + ''') + per_hour_result = per_hour_query.first() + # Convert to datetime + return { + 'start': datetime.fromtimestamp(min_max_result[0]), + 'end': datetime.fromtimestamp(min_max_result[1]), + 'count': min_max_result[2], + 'length_hours': (min_max_result[1] - min_max_result[0]) // 3600, + 'per_hour': per_hour_result[0], + } + + +def get_punch_card(session): + query = session.execute(''' + SELECT + CAST((expire_timestamp / 300) AS SIGNED) ts_date, + COUNT(*) how_many + FROM `sightings` + GROUP BY ts_date ORDER BY ts_date + ''') + results = [(i, r[1]) for (i, r) in enumerate(query.fetchall())] + return results + + +def get_top_pokemon(session, count=30, order='DESC'): + query = session.execute(''' + SELECT + pokemon_id, + COUNT(*) how_many + FROM sightings + GROUP BY pokemon_id + ORDER BY how_many {order} + LIMIT {count} + '''.format(order=order, count=count)) + return query.fetchall() + + +def get_stage2_pokemon(session): + result = [] + for pokemon_id in config.STAGE2: + count = session.query(Sighting) \ + .filter(Sighting.pokemon_id == pokemon_id) \ + .count() + if count > 0: + result.append((pokemon_id, count)) + return result + + +def get_nonexistent_pokemon(session): + result = [] + query = session.execute(''' + SELECT DISTINCT pokemon_id FROM sightings + ''') + db_ids = [r[0] for r in query.fetchall()] + for pokemon_id in range(1, 152): + if pokemon_id not in db_ids: + result.append(pokemon_id) + return result + + +def get_all_sightings(session, pokemon_ids): + # TODO: rename this and get_sightings + query = session.query(Sighting) \ + .filter(Sighting.pokemon_id.in_(pokemon_ids)) \ + .all() + return query diff --git a/templates/report.html b/templates/report.html new file mode 100644 index 0000000000..94c82eddda --- /dev/null +++ b/templates/report.html @@ -0,0 +1,182 @@ + + + + + Pokeminer Report + + + + + + + + + +
+

Pokeminer Report

+

Generated on {{ current_date.strftime('%Y-%m-%d %H:%M:%S') }}

+ +

This report contains statistics about data gathered during mining session for the city of {{ city }}.

+ +

During that session, {{ total_spawn_count }} Pokemon have been seen on an area of about {{ area }} square km. Data gathering started on {{ session_start.strftime('%Y-%m-%d %H:%M:%S') }} and ended on {{ session_end.strftime('%Y-%m-%d %H:%M:%S') }}, lasting {{ session_length_hours }} hours. There were {{ spawns_per_hour }} spawns per hour on average.

+ +

Below chart shows number of spawns seen per 5 minutes blocks:

+ +
+ +

Heatmap

+ +

All noticed spawn locations. The redder the point is, more Pokemon spawn there.

+ +

(will slow down browser!)

+ +
+ +

Most & least frequent species

+ +

Top 30 that spawned the most number of times during above period:

+ +
+ {% for icon in icons.top30 %} + + {% if loop.index > 0 and loop.index % 10 == 0 %} +
+ {% endif %} + {% endfor %} +
+ +
+ +

Bottom 30 that spawned the least number of times during above period, and all of their spawning places:

+ +
+ {% for icon in icons.bottom30 %} + + {% if loop.index > 0 and loop.index % 10 == 0 %} +
+ {% endif %} + {% endfor %} +
+ +
+ +
+ +

Evolutions and rare Pokemon

+ +

Stage 2 evolutions and Pokemon considered "rare", together with their spawning places:

+ +
+ {% for icon in icons.stage2 %} + + {% if loop.index > 0 and loop.index % 10 == 0 %} +
+ {% endif %} + {% endfor %} +
+ +
+ +
+ +

Nonexistent species

+ +

Those Pokemon didn't spawn during data gathering period:

+ +
+ {% for icon in icons.nonexistent %} + + {% if loop.index > 0 and loop.index % 10 == 0 %} +
+ {% endif %} + {% endfor %} +
+
+ + + diff --git a/web.py b/web.py index ee41f54b1b..3f9312a5c1 100644 --- a/web.py +++ b/web.py @@ -4,7 +4,7 @@ import json import requests -from flask import Flask, render_template +from flask import Flask, request, render_template from flask_googlemaps import GoogleMaps from flask_googlemaps import Map from flask_googlemaps import icons @@ -156,6 +156,82 @@ def get_map(): return fullmap +@app.route('/report') +def report_main(): + session = db.Session() + top_pokemon = db.get_top_pokemon(session) + bottom_pokemon = db.get_top_pokemon(session, order='ASC') + bottom_sightings = db.get_all_sightings( + session, [r[0] for r in bottom_pokemon] + ) + stage2_pokemon = db.get_stage2_pokemon(session) + stage2_sightings = db.get_all_sightings( + session, [r[0] for r in stage2_pokemon] + ) + js_data = { + 'charts_data': { + 'punchcard': db.get_punch_card(session), + 'top30': [(pokemon_names[str(r[0])], r[1]) for r in top_pokemon], + 'bottom30': [ + (pokemon_names[str(r[0])], r[1]) for r in bottom_pokemon + ], + 'stage2': [ + (pokemon_names[str(r[0])], r[1]) for r in stage2_pokemon + ], + }, + 'maps_data': { + 'bottom30': [sighting_to_marker(s) for s in bottom_sightings], + 'stage2': [sighting_to_marker(s) for s in stage2_sightings], + }, + 'map_center': utils.get_map_center(), + 'zoom': 12, + } + icons = { + 'top30': [(r[0], pokemon_names[str(r[0])]) for r in top_pokemon], + 'bottom30': [(r[0], pokemon_names[str(r[0])]) for r in bottom_pokemon], + 'stage2': [(r[0], pokemon_names[str(r[0])]) for r in stage2_pokemon], + 'nonexistent': [ + (r, pokemon_names[str(r)]) + for r in db.get_nonexistent_pokemon(session) + ] + } + session_stats = db.get_session_stats(session) + session.close() + return render_template( + 'report.html', + current_date=datetime.now(), + city=u'Wrocław', + area=96, + total_spawn_count=session_stats['count'], + spawns_per_hour=session_stats['per_hour'], + session_start=session_stats['start'], + session_end=session_stats['end'], + session_length_hours=int(session_stats['length_hours']), + js_data=js_data, + icons=icons, + ) + + +def sighting_to_marker(sighting): + return { + 'icon': '/static/icons/{}.png'.format(sighting.pokemon_id), + 'lat': sighting.lat, + 'lon': sighting.lon, + } + + +@app.route('/report/heatmap') +def report_heatmap(): + session = db.Session() + points = session.query(db.Sighting.lat, db.Sighting.lon) + pokemon_id = request.args.get('id') + if pokemon_id: + points = points.filter(db.Sighting.pokemon_id == int(pokemon_id)) + points = points.all() + session.close() + return json.dumps(points) + + if __name__ == '__main__': args = get_args() app.run(debug=True, threaded=True, host=args.host, port=args.port) From 61d894184d6b3c65ccb6ae9ecb84052501f0555a Mon Sep 17 00:00:00 2001 From: Modrzew Date: Sun, 24 Jul 2016 13:06:01 +0200 Subject: [PATCH 2/7] Bigger map and bigger zoom --- templates/report.html | 2 +- web.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/report.html b/templates/report.html index 94c82eddda..373bbc1b55 100644 --- a/templates/report.html +++ b/templates/report.html @@ -7,7 +7,7 @@ diff --git a/web.py b/web.py index 3f9312a5c1..31b58995ee 100644 --- a/web.py +++ b/web.py @@ -184,7 +184,7 @@ def report_main(): 'stage2': [sighting_to_marker(s) for s in stage2_sightings], }, 'map_center': utils.get_map_center(), - 'zoom': 12, + 'zoom': 11, } icons = { 'top30': [(r[0], pokemon_names[str(r[0])]) for r in top_pokemon], From 82db6eb5b8600dcb2177b98ebd635a209c6e9d0a Mon Sep 17 00:00:00 2001 From: Modrzew Date: Sun, 24 Jul 2016 13:17:13 +0200 Subject: [PATCH 3/7] Stylistic changes --- templates/report.html | 6 ++++-- web.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/templates/report.html b/templates/report.html index 373bbc1b55..cb7577dba9 100644 --- a/templates/report.html +++ b/templates/report.html @@ -7,7 +7,7 @@ @@ -35,7 +35,8 @@ ].concat(data)); var chart = new google.visualization.BarChart(document.getElementById(elemId)); chart.draw(combinedData, { - height: 1000 + height: 1000, + chartArea: {width: '70%', height: '80%'}, }); } @@ -46,6 +47,7 @@ var chart = new google.visualization.LineChart(document.getElementById('hourspunchardchart')); chart.draw(combinedData, { height: 300, + chartArea: {width: '70%', height: '80%'}, curveType: 'function', legend: {position: 'none'} }); diff --git a/web.py b/web.py index 31b58995ee..35455ca1b5 100644 --- a/web.py +++ b/web.py @@ -184,7 +184,7 @@ def report_main(): 'stage2': [sighting_to_marker(s) for s in stage2_sightings], }, 'map_center': utils.get_map_center(), - 'zoom': 11, + 'zoom': 13, } icons = { 'top30': [(r[0], pokemon_names[str(r[0])]) for r in top_pokemon], From a385d5d440a7403d3d3e75f142bbde6b4d79774e Mon Sep 17 00:00:00 2001 From: Modrzew Date: Sun, 24 Jul 2016 13:40:01 +0200 Subject: [PATCH 4/7] A bit of legalese --- templates/report.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/report.html b/templates/report.html index cb7577dba9..3b8843d2e6 100644 --- a/templates/report.html +++ b/templates/report.html @@ -176,6 +176,14 @@

Nonexistent species

{% endif %} {% endfor %} + +

Footnotes

+ +

This report was generated using pokeminer, a tool for gathering data about Pokemon Go.

+ +

Visit https://github.com/modrzew/pokeminer for more info.

+ +

This report is available under Creative Commons CC-BY-4.0 license: https://creativecommons.org/licenses/by/4.0/.