Merge branch 'simple-report'

modrzew · Jul 24, 2016 · 51f8fed · 51f8fed
2 parents 47b4c0a + fe88a45
commit 51f8fed
Show file tree

Hide file tree

Showing 4 changed files with 360 additions and 1 deletion.
diff --git a/db.py b/db.py
@@ -1,3 +1,5 @@
+from datetime import datetime
+import json
 import time
 
 from sqlalchemy.orm import sessionmaker
@@ -7,6 +9,10 @@
 from sqlalchemy.sql import not_
 
 
+with open('locales/pokemon.en.json') as f:
+    pokemon_names = json.load(f)
+
+
 try:
     import config
     DB_ENGINE = config.DB_ENGINE
@@ -69,3 +75,84 @@ def get_sightings(session):
     if trash_list:
         query = query.filter(not_(Sighting.pokemon_id.in_(config.TRASH_IDS)))
     return query.all()
+
+
+def get_session_stats(session):
+    min_max_query = session.execute('''
+        SELECT
+            MIN(expire_timestamp) ts_min,
+            MAX(expire_timestamp) ts_max,
+            COUNT(*)
+        FROM `sightings`;
+    ''')
+    min_max_result = min_max_query.first()
+    length_hours = (min_max_result[1] - min_max_result[0]) // 3600
+    # Convert to datetime
+    return {
+        'start': datetime.fromtimestamp(min_max_result[0]),
+        'end': datetime.fromtimestamp(min_max_result[1]),
+        'count': min_max_result[2],
+        'length_hours': length_hours,
+        'per_hour': min_max_result[2] / length_hours,
+    }
+
+
+def get_punch_card(session):
+    query = session.execute('''
+        SELECT
+            CAST((expire_timestamp / 300) AS SIGNED) ts_date,
+            COUNT(*) how_many
+        FROM `sightings`
+        GROUP BY ts_date ORDER BY ts_date
+    ''')
+    results = query.fetchall()
+    results_dict = {r[0]: r[1] for r in results}
+    filled = []
+    for row_no, i in enumerate(range(int(results[0][0]), int(results[-1][0]))):
+        item = results_dict.get(i)
+        filled.append((row_no, item if item else 0))
+    return filled
+
+
+def get_top_pokemon(session, count=30, order='DESC'):
+    query = session.execute('''
+        SELECT
+            pokemon_id,
+            COUNT(*) how_many
+        FROM sightings
+        GROUP BY pokemon_id
+        ORDER BY how_many {order}
+        LIMIT {count}
+    '''.format(order=order, count=count))
+    return query.fetchall()
+
+
+def get_stage2_pokemon(session):
+    result = []
+    for pokemon_id in config.STAGE2:
+        count = session.query(Sighting) \
+            .filter(Sighting.pokemon_id == pokemon_id) \
+            .count()
+        if count > 0:
+            result.append((pokemon_id, count))
+    return result
+
+
+def get_nonexistent_pokemon(session):
+    result = []
+    query = session.execute('''
+        SELECT DISTINCT pokemon_id FROM sightings
+    ''')
+    db_ids = [r[0] for r in query.fetchall()]
+    for pokemon_id in range(1, 152):
+        if pokemon_id not in db_ids:
+            result.append(pokemon_id)
+    return result
+
+
+def get_all_sightings(session, pokemon_ids):
+    # TODO: rename this and get_sightings
+    query = session.query(Sighting) \
+        .filter(Sighting.pokemon_id.in_(pokemon_ids)) \
+        .all()
+    return query
diff --git a/templates/report.html b/templates/report.html
@@ -0,0 +1,194 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Pokeminer Report</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" integrity="sha384-1q8mTJOASx8j1Au+a5WDVnPi2lkFfwwEAa8hDDdjZlpLegxhjVME1fgjWPGmkzs7" crossorigin="anonymous">
+
+    <style>
+        .map {
+            height: 1000px;
+        }
+    </style>
+
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.0/jquery.min.js"></script>
+    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+    <script type="text/javascript">
+        google.charts.load('current', {'packages':['corechart']});
+        google.charts.setOnLoadCallback(drawCharts);
+
+        var backendData = {{ js_data|tojson }};
+
+        function drawCharts () {
+            drawHoursPunchcardChart(backendData.charts_data.punchcard);
+            drawPokemonChart(backendData.charts_data.top30, 'top30chart');
+            drawPokemonChart(backendData.charts_data.bottom30, 'bottom30chart');
+            drawPokemonChart(backendData.charts_data.stage2, 'stage2chart');
+        }
+
+        function drawPokemonChart (data, elemId) {
+            if (!data.length) {
+                return;
+            }
+            var combinedData = google.visualization.arrayToDataTable([
+                ['Pokemon', 'Spawns seen'],
+            ].concat(data));
+            var chart = new google.visualization.BarChart(document.getElementById(elemId));
+            chart.draw(combinedData, {
+                height: 1000,
+                chartArea: {width: '70%', height: '80%'},
+            });
+        }
+
+        function drawHoursPunchcardChart (data) {
+            var combinedData = google.visualization.arrayToDataTable([
+                ['Time', 'Spawns seen'],
+            ].concat(data));
+            var chart = new google.visualization.LineChart(document.getElementById('hourspunchardchart'));
+            chart.draw(combinedData, {
+                height: 300,
+                chartArea: {width: '70%', height: '80%'},
+                curveType: 'function',
+                legend: {position: 'none'}
+            });
+        }
+
+        var maps = {heat: null, bottom30: null, stage2: null};
+
+        function initMaps() {
+            Object.keys(maps).forEach(function (key) {
+                maps[key] = new google.maps.Map(document.getElementById(key + 'map'), {
+                    zoom: backendData.zoom,
+                    center: {lat: backendData.map_center[0], lng: backendData.map_center[1]},
+                    disableDefaultUI: true,
+                });
+                // Add markers
+                if (typeof backendData.maps_data[key] !== 'undefined') {
+                    backendData.maps_data[key].forEach(function (item) {
+                        new google.maps.Marker({
+                            position: new google.maps.LatLng(item.lat, item.lon),
+                            icon: item['icon'],
+                            map: maps[key],
+                        });
+                    });
+                }
+            });
+        }
+
+        function displayHeatmap (points) {
+            var layer = new google.maps.visualization.HeatmapLayer({
+              data: points
+            });
+            layer.setMap(maps.heat);
+        }
+
+        $(function () {
+            var heatmapPoints;
+            $.get('/report/heatmap').done(function (result) {
+                heatmapPoints = JSON.parse(result).map(function (elem) {
+                    return new google.maps.LatLng(elem[0], elem[1]);
+                });
+            });
+            $('#displayHeatmap').on('click', function () {
+                displayHeatmap(heatmapPoints);
+                $(this).parent().remove();
+            });
+        });
+    </script>
+</head>
+<body>
+    <div class="container">
+        <h1>Pokeminer Report</h1>
+        <h4>Generated on {{ current_date.strftime('%Y-%m-%d %H:%M:%S') }}</h4>
+
+        <p><b>Disclaimer:</b> data may be incomplete due to various issues that might have happened (bugs, unstable servers, bugs on the servers etc.). If there is data about a sighting of a Pokemon in given location at particular time, almost certainly such spawning happened. On the other hand, there is no guarantee that Pokeminer database contains <i>all</i> spawnings, so there might be wild Pokemon not contained in this report. Your mileage may vary.</p>
+
+        <p>This report contains statistics about data gathered during mining session for the city of {{ city }}.</p>
+
+        <p>During that session, <b> {{ total_spawn_count }}</b> Pokemon have been seen on an area of about <b>{{ area }} square km</b>. Data gathering started on <b>{{ session_start.strftime('%Y-%m-%d %H:%M:%S') }}</b> and ended on <b>{{ session_end.strftime('%Y-%m-%d %H:%M:%S') }}</b>, lasting <b>{{ session_length_hours }} hours</b>. There were {{ spawns_per_hour }} spawns per hour on average.</p>
+
+        <p>Below chart shows number of spawns seen per 5 minutes blocks:</p>
+
+        <div id="hourspunchardchart"></div>
+
+        <h3>Heatmap</h3>
+
+        <p>All noticed spawn locations. The redder the point is, more Pokemon spawn there.</p>
+
+        <p><button id="displayHeatmap">Display heatmap</button> (will slow down browser!)</p>
+
+        <div id="heatmap" class="map"></div>
+
+        <h3>Most &amp; least frequently spawning species</h2>
+
+        <p><b>Top 30</b> that spawned the most number of times during above period:</p>
+
+        <div class="text-center">
+            {% for icon in icons.top30 %}
+                <img src="../static/larger-icons/{{ icon[0] }}.png" title="#{{ icon[0] }} {{ icon[1] }}">
+                {% if loop.index > 0 and loop.index % 10 == 0 %}
+                    <br>
+                {% endif %}
+            {% endfor %}
+        </div>
+
+        <div id="top30chart"></div>
+
+        <p><b>Bottom 30</b> that spawned the least number of times during above period, and all of their spawning places:</p>
+
+        <div class="text-center">
+            {% for icon in icons.bottom30 %}
+                <img src="../static/larger-icons/{{ icon[0] }}.png" title="#{{ icon[0] }} {{ icon[1] }}">
+                {% if loop.index > 0 and loop.index % 10 == 0 %}
+                    <br>
+                {% endif %}
+            {% endfor %}
+        </div>
+
+        <div id="bottom30chart"></div>
+
+        <div id="bottom30map" class="map"></div>
+
+        <h3>Evolutions and rare Pokemon</h3>
+
+        <p><b>Stage 2 evolutions</b> and Pokemon subjectively considered "rare" by author of this report, together with their spawning places:</p>
+
+        <div class="text-center">
+            {% for icon in icons.stage2 %}
+                <img src="../static/larger-icons/{{ icon[0] }}.png" title="#{{ icon[0] }} {{ icon[1] }}">
+                {% if loop.index > 0 and loop.index % 10 == 0 %}
+                    <br>
+                {% endif %}
+            {% endfor %}
+        </div>
+
+        <div id="stage2chart"></div>
+
+        <div id="stage2map" class="map"></div>
+
+        <h3>Nonexistent species</h3>
+
+        <p>Those Pokemon didn't spawn during data gathering period:</p>
+
+        <div class="text-center">
+            {% for icon in icons.nonexistent %}
+                <img src="../static/larger-icons/{{ icon[0] }}.png" title="#{{ icon[0] }} {{ icon[1] }}">
+                {% if loop.index > 0 and loop.index % 10 == 0 %}
+                    <br>
+                {% endif %}
+            {% endfor %}
+        </div>
+
+        <h2>Footnotes</h2>
+
+        <p>This report was generated using pokeminer, a tool for gathering data about Pokemon Go.</p>
+
+        <p>Visit <a href="https://github.com/modrzew/pokeminer">https://github.com/modrzew/pokeminer</a> for more info.</p>
+
+        <p>This report is available under Creative Commons CC-BY-4.0 license: <a href="https://creativecommons.org/licenses/by/4.0/">https://creativecommons.org/licenses/by/4.0/</a>.</p>
+    </div>
+    <script async defer
+        src="https://maps.googleapis.com/maps/api/js?key=AIzaSyAy8rKI_g-dgYf9f7tMOafbNypeaJJvVeA&amp;libraries=visualization&amp;callback=initMaps">
+    </script>
+</body>
+</html>
diff --git a/web.py b/web.py
@@ -4,7 +4,7 @@
 import json
 
 import requests
-from flask import Flask, render_template
+from flask import Flask, request, render_template
 from flask_googlemaps import GoogleMaps
 from flask_googlemaps import Map
 from flask_googlemaps import icons
@@ -156,6 +156,82 @@ def get_map():
     return fullmap
 
 
+@app.route('/report')
+def report_main():
+    session = db.Session()
+    top_pokemon = db.get_top_pokemon(session)
+    bottom_pokemon = db.get_top_pokemon(session, order='ASC')
+    bottom_sightings = db.get_all_sightings(
+        session, [r[0] for r in bottom_pokemon]
+    )
+    stage2_pokemon = db.get_stage2_pokemon(session)
+    stage2_sightings = db.get_all_sightings(
+        session, [r[0] for r in stage2_pokemon]
+    )
+    js_data = {
+        'charts_data': {
+            'punchcard': db.get_punch_card(session),
+            'top30': [(pokemon_names[str(r[0])], r[1]) for r in top_pokemon],
+            'bottom30': [
+                (pokemon_names[str(r[0])], r[1]) for r in bottom_pokemon
+            ],
+            'stage2': [
+                (pokemon_names[str(r[0])], r[1]) for r in stage2_pokemon
+            ],
+        },
+        'maps_data': {
+            'bottom30': [sighting_to_marker(s) for s in bottom_sightings],
+            'stage2': [sighting_to_marker(s) for s in stage2_sightings],
+        },
+        'map_center': utils.get_map_center(),
+        'zoom': 13,
+    }
+    icons = {
+        'top30': [(r[0], pokemon_names[str(r[0])]) for r in top_pokemon],
+        'bottom30': [(r[0], pokemon_names[str(r[0])]) for r in bottom_pokemon],
+        'stage2': [(r[0], pokemon_names[str(r[0])]) for r in stage2_pokemon],
+        'nonexistent': [
+            (r, pokemon_names[str(r)])
+            for r in db.get_nonexistent_pokemon(session)
+        ]
+    }
+    session_stats = db.get_session_stats(session)
+    session.close()
+    return render_template(
+        'report.html',
+        current_date=datetime.now(),
+        city=u'Wrocław',
+        area=96,
+        total_spawn_count=session_stats['count'],
+        spawns_per_hour=session_stats['per_hour'],
+        session_start=session_stats['start'],
+        session_end=session_stats['end'],
+        session_length_hours=int(session_stats['length_hours']),
+        js_data=js_data,
+        icons=icons,
+    )
+
+
+def sighting_to_marker(sighting):
+    return {
+        'icon': '/static/icons/{}.png'.format(sighting.pokemon_id),
+        'lat': sighting.lat,
+        'lon': sighting.lon,
+    }
+
+
+@app.route('/report/heatmap')
+def report_heatmap():
+    session = db.Session()
+    points = session.query(db.Sighting.lat, db.Sighting.lon)
+    pokemon_id = request.args.get('id')
+    if pokemon_id:
+        points = points.filter(db.Sighting.pokemon_id == int(pokemon_id))
+    points = points.all()
+    session.close()
+    return json.dumps(points)
+
+
 if __name__ == '__main__':
     args = get_args()
     app.run(debug=True, threaded=True, host=args.host, port=args.port)
diff --git a/worker.py b/worker.py
@@ -561,6 +561,8 @@ def process_step(
             break
 
     for poke in visible:
+        if poke.TimeTillHiddenMs < 10 * 1000:
+            continue
         disappear_timestamp = time.time() + poke.TimeTillHiddenMs / 1000
         pokemons[poke.SpawnPointId] = {
             'lat': poke.Latitude,