-
Notifications
You must be signed in to change notification settings - Fork 0
/
ChicagoRealTimeAQ.html
932 lines (706 loc) · 55.5 KB
/
ChicagoRealTimeAQ.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
<!DOCTYPE html>
<!--[if lt IE 9 ]><html class="no-js oldie" lang="en"> <![endif]-->
<!--[if IE 9 ]><html class="no-js oldie ie9" lang="en"> <![endif]-->
<!--[if (gte IE 9)|!(IE)]><!-->
<html class="no-js" lang="en">
<!--<![endif]-->
<head>
<!--- basic page needs
================================================== -->
<meta charset="utf-8">
<title>Chicago Real-Time Air Quality</title>
<meta name="description" content="">
<meta name="author" content="">
<!-- mobile specific metas
================================================== -->
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- CSS
================================================== -->
<link rel="stylesheet" href="/css/base.css">
<link rel="stylesheet" href="/css/vendor.css">
<link rel="stylesheet" href="/css/main.css">
<!-- script
================================================== -->
<script src="/js/modernizr.js"></script>
<script src="/js/pace.min.js"></script>
<!-- favicons
================================================== -->
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
<link rel="icon" href="favicon.ico" type="image/x-icon">
</head>
<body id="top">
<!-- header
================================================== -->
<header class="s-header">
<div class="header-logo">
<a class="site-logo" href="/index.html">
<img src="/images/logo.png" alt="Homepage">
</a>
</div>
<nav class="header-nav">
<a href="#0" class="header-nav__close" title="close"><span>Close</span></a>
<div class="header-nav__content">
<h3>Navigation</h3>
<ul class="header-nav__list">
<li class="current"><a href="/index.html" title="home">Home</a></li>
<li><a class="smoothscroll" href="#about" title="about">About</a></li>
<li><a class="smoothscroll" href="#articles" title="services">Documentation</a></li>
<li><a class="smoothscroll" href="#video" title="works">Video</a></li>
<li><a class="smoothscroll" href="#contact" title="contact">Contact</a></li>
</ul>
<p>Quick Menu</p>
<ul class="header-nav__social">
<li>
<a href="https://www.facebook.com/mrkmantovani"><i class="fa fa-facebook" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://www.linkedin.com/in/mirkomantovani/"><i class="fa fa-linkedin" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://www.instagram.com/mirkomantovani/"><i class="fa fa-instagram" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://play.google.com/store/apps/developer?id=Mirko+Mantovani"><i class="fa fa-android" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://github.com/mirkomantovani"><i class="fa fa-github" aria-hidden="true"></i></a>
</li>
</ul>
</div> <!-- end header-nav__content -->
</nav> <!-- end header-nav -->
<a class="header-menu-toggle" href="#0">
<span class="header-menu-text">Menu</span>
<span class="header-menu-icon"></span>
</a>
</header> <!-- end s-header -->
<!-- home
================================================== -->
<section id="home" class="s-home target-section" data-parallax="scroll" data-image-src="/images/p3viz/cover.png" data-natural-width=3000 data-natural-height=2000 data-position-y=center>
<div class="overlay"></div>
<div class="shadow-overlay"></div>
<div class="home-content">
<div class="row home-content__main">
<h3>Project page</h3>
<h1>
Chicago Real-Time Air Quality, Visualization and Visual Analytics
</h1>
<div class="home-content__buttons">
<a href="#video" class="smoothscroll btn btn--stroke">
Video
</a>
<a href="#about" class="smoothscroll btn btn--stroke">
About
</a>
</div>
</div>
<div class="home-content__scroll">
<a href="#about" class="scroll-link smoothscroll">
<span>Scroll Down</span>
</a>
</div>
<div class="home-content__line"></div>
</div> <!-- end home-content -->
<ul class="home-social">
<li>
<a href="https://www.facebook.com/mrkmantovani"><i class="fa fa-facebook" aria-hidden="true"></i><span>Facebook</span></a>
</li>
<li>
<a href="https://www.linkedin.com/in/mirkomantovani/"><i class="fa fa-linkedin" aria-hidden="true"></i><span>LinkedIn</span></a>
</li>
<li>
<a href="https://www.instagram.com/mirkomantovani/"><i class="fa fa-instagram" aria-hidden="true"></i><span>Instagram</span></a>
</li>
<li>
<a href="https://play.google.com/store/apps/developer?id=Mirko+Mantovani"><i class="fa fa-android" aria-hidden="true"></i><span>PlayStore</span></a>
</li>
<li>
<a href="https://github.com/mirkomantovani"><i class="fa fa-github" aria-hidden="true"></i><span>GitHub</span></a>
</li>
</ul>
<!-- end home-social -->
</section> <!-- end s-home -->
<!-- about
================================================== -->
<section id='about' class="s-about">
<div class="row section-header has-bottom-sep" data-aos="fade-up">
<div class="col-full">
<h3 class="subhead subhead--dark">About</h3>
<h1 class="display-1 display-1--light">Chicago Real-Time Air Quality, Visualization and Visual Analytics</h1>
</div>
</div> <!-- end section-header -->
<div class="row about-desc" data-aos="fade-up">
<div class="col-full">
<p>
<h2>Introduction</h2>
This is the third project documentation for the CS 424 Visualization and Visual Analytics class at UIC. It consists in Chicago real-time air quality and weather data visualizations in an interactive web application created using the Shiny R library.
<br><br>
<h2>Authors</h2>
<a href="https://mmanto2.people.uic.edu/" style="color: darkblue">Mirko Mantovani</a><br>
<a href="https://avasud8.people.uic.edu/" style="color: darkblue">Abhishek Vasudevan</a><br>
<a href="https://ashwanikhemani.github.io/" style="color: darkblue">Ashwani Khemani</a>
<h2>Access the application</h2>
<a href="http://shiny.evl.uic.edu:3838/p3g4/Chicago-RealTime-AQ" style="text-decoration: underline">Access the Shiny application on a
local Shiny server</a>
<h2>Data</h2>
<a href='https://arrayofthings.github.io/' target='_blank'>Array of Things</a><br>
<a href='https://openaq.org/' target='_blank'>Open Air Quality</a><br>
<a href='https://darksky.net/' target='_blank'>Dark Sky</a><br>
<a href='https://dev.socrata.com/foundry/data.cityofchicago.org/n4j6-wkkf'>Chicago Traffic Tracker</a>
<h2>Code</h2>
<a href="https://github.com/mirkomantovani/Chicago-RealTime-AQ" style="text-decoration: underline">Go to Github Repository</a>
</div>
</div> <!-- end about-desc -->
<div class="about__line"></div>
</section> <!-- end s-about -->
<section id="articles" class="s-clients">
<div class="row about-desc" data-aos="fade-up">
<div class="col-full">
<h1 class="display-2">How to run the application</h1>
<h2>Accessing the web application online</h2>
<p>
Just click on the provided link to access the web application hosted on the EVL server from your browser. It is recommended to use a recent version of Google Chrome.
</p>
<h2>Hosting the web application locally</h2>
Clone the project repository. Open app.R with RStudio, download <a href="https://www.rstudio.com/" style="text-decoration: underline">RStudio</a> and <a href="https://www.r-project.org/" style="text-decoration: underline">R</a> if you don't have them installed on your machine. Download all the R libraries used in the project executing the command install.packages("library") in the RStudio console. The application. The application queires data realtime from the various data sources Array of Things, Darksky , Open AQ and Chicago Traffic Tracker in order to perform various visualizations. Please follow the below steps to do the required preprocessing before running the application to generate the data again.
<ul>
<li>There is 1 preprocessing required to generate the necessary data file for the openAQ part of the application. Essentially, what this script does is to generate latitude and longitude pairs for all the openAQ locations in the city of Chicago. This is stored as an fst file which will be loaded into the app when it is run.
</li>
<li> The file is generated by running the preprocess_aqi.R which would create an fst file in the following location: fst/openaq.fst
</li>
</ul>
Once we have all the neceessary data/fst files , we can now proceed with running of the application.
Run the application via RStudio by clicking Run App at the top right on the main RStudio panel. Access it using a browser and with the local machine address and port on which the application is started. (<a href="http://127.0.0.1:port_num/" style="text-decoration: underline">http://127.0.0.1:port/</a>)
<br><br>
<h1 class="display-2">Functionalities</h1>
<h2>Getting started and Sidebar</h2>
The application starts in full screen. You can open the menu items for different categories in the sidebar to see more options. The inputs item helps the user to switch between metric or imperial units for the whole application , to enable or disable the heatmap visualization and to show/hide the sensor nodes in the tabular form.
<h2>Map</h2>
This application has only one main page, a full screen map centered in the Chicago area. On the map there are sensor nodes that can be clicked by the user to get real-time, as well as historical data from it. There is also real-time traffic data that can also be clicked on to get more information.
As the rest of the application, this map is responsive and particularly, it was created for the big wall of display that we have in the classroom (11520 by 3240 pixels). In order to allow a practical user experience in terms of the touch screen wall present in the classroom, the UI has been designed to work best in this configuration that is the one applied to this version of the application:
<h4>Nodes</h4>
Nodes are groups of various hardware sensors located in the Chicago area. Each node has multiple sensors in it.
The nodes come from 2 different sources: <a href="https://arrayofthings.github.io/">Array of Things</a> and <a href="https://openaq.org/">Open Air Quality</a>. The AoT nodes are blue, the OpenAQ green. The red nodes are the inactive nodes from AoT. Inactive nodes are all the nodes which are deployed but when queried respond with no observation for each sensor they have.<br>
By clicking on any node, a popup will appear showing more information about the node. The node is also queried and by waiting a few seconds it is possible to visualize the data it provides in the plot panel. We also query at the same time the <a href="https://darksky.net/">Dark Sky</a> APIs that retrieve weather data and other measures for any node and they are displayed in the plots panel as well. Keep in mind that it could take a while to load they are real time data. Especially for the last 7 days and 24 hours the data gathering and preprocessing is really slow, because multiple requests are performed and in particular the AoT APIs have a very slow response time.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/2.png" alt="Screen" class="responsive">
</div>
<h4>Nodes Table</h4>
The different Array of Things nodes along with their location are shown in tabular form on the left. The table can be shown and hidden based on the user preference. The table show the information about the various AoT nodes, what measures they are reporting and the node overall status active or inactive. The nodes which are reporting measures have "True" corresponding for that measure. The user can filter using the checkboxes below in the panel to show and hide sites based on the measures selected. The user can select any node from the table and visualize the measures for that site in form of graphical and tabular form. The comparison for two different sites can be done by selecting one row at a time and then the visualization for them show up in the current and previous section in the graphical and tabular panel.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/9.png" alt="Screen" class="responsive">
</div>
<h4>Map background</h4>
The map background can be changed by clicking on the select boxes in the layers menu. There are 4 different map backgrounds. The default is a drawn map with train stations, streets and highways, parks and icons for most important locations.<br>
The Dark Matter allows to make good use of the different colors (traffic and nodes) that pop out on the black background.<br>
The satellite is a usual satellite image of Chicago.
The terrain is a mix of all the previous features: allows to better visualize colors given that the predominant color is white/grey, it has terrain features (see the green areas and water), it has streets and highways highlighted.
<div class="row clients-testimonials" data-aos="fade-up">
<div class="col-full">
<h1>Map backgrounds</h1>
<div class="testimonials">
<div class="testimonials__slide">
<h1 class="display-2">Default</h1>
<div class="testimonials__info">
<!-- <span class="testimonials__name">Room I view</span>-->
</div>
<br><br>
<img src="/images/p3viz/3.png" alt="Screen" class="responsive">
<br><br>
</div>
<div class="testimonials__slide">
<h1 class="display-2">Dark matter</h1>
<div class="testimonials__info">
<!-- <span class="testimonials__name">Room I view</span>-->
</div>
<br><br>
<img src="/images/p3viz/4.png" alt="Screen" class="responsive">
<br><br>
</div>
<div class="testimonials__slide">
<h1 class="display-2">Satellite</h1>
<div class="testimonials__info">
<!-- <span class="testimonials__name">Room I view</span>-->
</div>
<br><br>
<img src="/images/p3viz/5.png" alt="Screen" class="responsive">
<br><br>
</div>
<div class="testimonials__slide">
<h1 class="display-2">Terrain</h1>
<div class="testimonials__info">
<!-- <span class="testimonials__name">Room I view</span>-->
</div>
<br><br>
<img src="/images/p3viz/6.png" alt="Screen" class="responsive">
<br><br>
</div>
</div>
</div>
</div>
<h4>Traffic Data</h4>
Traffic data come from <a href="https://dev.socrata.com/foundry/data.cityofchicago.org/n4j6-wkkf">Chicago Traffic Tracker</a>.
This data is real-time (at most 20 minutes ago) data that describe the congestion status in the streets of Chicago.
In particular, the dataset contains the current estimated speed for about 1250 segments covering 300 miles of arterial roads.<br>
We decided to integrate traffic data because it could help explain and give a cause to the real-time variations of pollutants such as CO, PM2.5, PM10, NO2 that are also produced by vehicles.
The roads are colored followed usual traffic color used in common GPS navigation applications (e.g. Google Maps). The colors conventions are: grey, no traffic (no cars or no data provided), blue: normal/high speed and above, yellow: medium speed, orange: low speed, red: very low or zero speed.<br>
Each road is clickable and more information are provided in a popup.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/7.png" alt="Screen" class="responsive">
</div>
<h2>Heatmap</h2>
The Heatmap can be use be used to visualize the various measure intensity across the city of Chicago. The Heatmap has three inputs : Type of Measure, Value Type (min,max,average) and time range (current, last 24 hours and last 7 days). The user can select any measure from the dropdown for the three data sources and visualize it for various values type and time duration. The Heatmap is created by interpolation all the nodes data for a particular data source in order to get better data for visualization for the entire city of chicago. Based on the amount of data available, the interpolation is not possible for some of the measures if we have very less number of nodes reporting for them and a message is shown to the user on the screen. There is a legend available for the heatmap which tell about the value being shown on the heatmap.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/8.png" alt="Screen" class="responsive">
</div>
<h2>Panels and menus</h2>
<h4>Plots panel</h4>
This is the most important panel of the application. It shows the data queried by the various nodes in the form of line charts or tabular form.
The inputs present in this panel are two: the first one is the time range, which can be selected from current (or most recent data), last 24 hours and last 7 days data. The second one is a set of checkboxes that allow the users to select only the pollutants features that they want to visualize.<br>
It is divided into two tabs: First tab lists the pollutants metrics and the second one lists weather/climate metrics.<br>
Each tab consists of a graphical and tabular view. The graphical view displays the metrics for node which is currently selected and it also shows the graph for the node which was previously selected. Similarly, the tabular view shows metrics for currently selected node in a data table and it also shows the the data table for previously selected node.<br/>
<b>Tab 1 - Pollutants</b><br/>
The following are the metrics which can be displayed in the first tab:
<br/>
1)CO, 2) H2S, 3) NO2, 4)O3, 5)SO2, 6)PM2.5, 7) PM10, 8) BC
<br/>
The metrics can be filtered/removed from the graph using the checkbox given at the end of the second graph output. Following screenshot shows a sample graph shown in the first tab:
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/tab1_graph.png" alt="Screen" class="responsive"> <!-- ADD IMAGE -->
</div>
The legend shows what colors are used for each metric along with the units (note that imperial units can be triggered from the main panel on the left). Distinct colors are used for each metric. Similarly, the same data can be viewed in tabular format:
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/tab1_table.png" alt="Screen" class="responsive"> <!-- ADD IMAGE -->
</div>
<b>Tab 2 - Weather measures</b><br/>
Similar to tab 1, we have graphs and tables for tab 2. The following are the measures which can be displayed for tab 2:<br/>
1)Temperature, 2) Humidity, 3) Intensity, 4) Wind Speed, 5) Cloud coverage, 6) Visibility, 7) Pressure, 8) Ozone<br/>
Note that temperature, humidity and intensity are available in both darksky and Array of Things data, hence the distinction used in the legend is that AoT measures have the trailing text "(AOT)" in their name while darksky measures do not have any trailing text. This can be seen in the example graph below:
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/tab2_graph.png" alt="Screen" class="responsive"> <!-- ADD IMAGE -->
</div>
As usual, you can convert the units to imperial and this is showed in the legend text. We can also visualize the data in tabular format as shown by the example table shown below. As there are many weather metrics, this tab is best visualized by filtering out variabls using the checkboxes at the end of the panel.<br/>
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/tab2_table.png" alt="Screen" class="responsive"> <!-- ADD IMAGE -->
</div>
<b>NOTE</b> that all these graphs and tables have data which is real time and the data is refreshed every minute. No data is stored locally. Even for displaying data for previously selected node's output graph/table, the data is fetched real time.
<h4>Layers menu</h4>
The layers menu can be opened by hovering over or clicking on the corresponding menu on the top right corner.
From there you can select one of the 4 different map backgrounds, as well as toggle on or off the sensors belonging to the nodes on the map based on the type of measure they give, or if they are active or inactive.
The last toggle is the traffic data, that shows or hides the roads real-time traffic visualization layer.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/11.png" alt="Screen" class="responsive">
</div>
<h2>Units of measure</h2>
To allow users from both the United States and the rest of the world to give a meaning to all of this data we provided a practical switch on the sidebar under the inputs tabs so that the User can choose to convert data from Metric to Imperial and viceversa.
Pollutants having the ppm unit (part per million) do not have any equivalent in the imperial system and hence remain the same. The conversion details for other pollutants are listed below:
<INSERT LIST>
<LIST IN NOTEBOOK>
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/12.png" alt="Screen" class="responsive">
</div>
<h2>Used R libraries</h2>
This is the list of R libraries used for this project:
<ul>
<li>shiny</li>
<li>devtools</li>
<li>shinydashboard</li>
<li>ggplot2</li>
<li>scales</li>
<li>shinythemes</li>
<li>dashboardthemes</li>
<li>ggthemes</li>
<li>shinyalert</li>
<li>leaflet</li>
<li>rgdal</li>
<li>shinyWidgets</li>
<li>viridis</li>
<li>cdlTools</li>
<li>htmltools</li>
<li>RColorBrewer</li>
<li>reshape2</li>
<li>fst</li>
<li>data.table</li>
<li>dplyr</li>
<li>tidyr</li>
<li>lubridate</li>
<li>tidyverse</li>
<li>ropenaq</li>
<li>darksky</li>
<li>RSocrata</li>
<li>base</li>
<li>sp</li>
<li>raster</li>
<li>gstat</li>
</ul>
<h1 class="display-2">Interesting Insights</h1>
<h4>Using traffic data</h4>
As explain in the documentation, using traffic data could help explain and give a cause to the real-time variations of pollutants such as CO, PM2.5, PM10, NO2 that are also produced by vehicles. A strategy to do this kind of visual analytics with the application would be to select in the layers the nodes that give information about those pollutants, then select nodes with different type of traffic and compare the current pollutants value. Unfortunately, the data provided by the nodes is not really good, the CO measurement is most of the times negative (and the APIs don't explain what could be the meaning of negative values, it could be that it means that it is 0), the NO2 is 0 everywhere in Chicago, and the pm2.5 and pm10 are almost not provided by any sensor. For the sake of explaining the application's functionalities, I provide a screenshot of what the application looks like when interactions of this type are performed.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/ii1.png" alt="Screen" class="responsive">
</div>
<h4>Last 7 days, downtown vs uptown</h4>
I tried to compare 2 different locations in terms of traffic, buildings, amount of people, closeness to the lake.
The upper plot shows the data for the last 7 days for a node located in uptown, close to the lake. The bottom plot shows data for the last 7 days for a node located in downtown, a little more west (further from the lake).
The things that I noticed are that: downtown the humidity level is more constant and overall less than uptown. H2S is only present downtown (it is 0 in the 08F node), the temperature is higher in uptown.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/ii2.png" alt="Screen" class="responsive">
</div>
<h4>Heatmap for Particulate Matter 2.5 over the week</h4>
I tried comparing the average PM2.5 measurement value from the OpenAQ source for the last week and it looks like that south side of Chicago has more PM2.5 particles as compare to north side of Chicago. The reason might be due to presence of scrap yards, distribution warehouses and low income neighborhoods in south part of chicago.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/openaq_pm2.5_last7.png" alt="Screen" class="responsive">
</div>
<h4>Heatmap for Light Intensity over the last week</h4>
I tried comparing the light intensity for Chicago area for the last week. It can be seen that few areas have a compartively very high light intensity as compared to the rest of the city. It may be due to the fact that some locations have better access to light as compared to other places leading to better intensity for those locations.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/light_intensity_7days.png" alt="Screen" class="responsive">
</div>
<h4>Temperature over the last week using Darksky</h4>
I tried comparing the Temperature values for the last weeek using Darksky API using the Heatmap and sometimes darksky shows absurd values for Temperature (-60 F) like shown in the figure. The reason behind this might be faulty sensors. I came to know by reading more that darksky sensors are prone to errors and can give wrong measurement sometimes.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/faulty_sensors.png" alt="Screen" class="responsive">
</div>
<h4>Data obtained fromopenAQ</h4>
OpenAQ reports more or less the same level of values for the weather measures. This is possibly because the weather is constant throughout Chicago or it is possible that the sensors report inaccurate values. The following table shows two openAQ nodes which have almost same values for many of the metrics.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/openaq_table.png" alt="Screen" class="responsive">
</div>
<h4>Comparing data from AoT and openAQ</h4>
For two locations which are geographically close, the values obtained for the pollutant Ozone is pretty different as seen from the data obtained from openAQ(top graph) and the bottom one(AoT). The measure is reported in ppm and hence even a small difference is a lot.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/openaq_2.png" alt="Screen" class="responsive">
</div>
<h4>Temperature obtained from Darksky and AoT</h4>
For a given AoT node, the darksky temperature is messed up as it shows -56 degrees celsius as shown in screenshot. On the other hand, the AoT sensor reports it accurately for the past 7 days.
<div class="testimonials__slide">
<div class="testimonials__info">
</div>
<img src="/images/p3viz/temperature_darksky.png" alt="Screen" class="responsive">
</div>
<!--
<div class="row clients-testimonials" data-aos="fade-up">
<div class="col-full">
<div class="testimonials">
<div class="testimonials__slide">
<div class="testimonials__info">
<span class="testimonials__name">AQI 1998</span>
</div>
<br><br>
<img src="/images/p2viz/interesting1.png" alt="Screen" class="responsive">
<br><br>
</div>
<div class="testimonials__slide">
<div class="testimonials__info">
<span class="testimonials__name">AQI 2018</span>
</div>
<br><br>
<img src="/images/p2viz/interesting2.png" alt="Screen" class="responsive">
<br><br>
</div>
</div>
</div>
</div>
-->
</div>
</div>
<div class="row about-desc" data-aos="fade-up">
<div class="col-full">
<!-- Templates:
Picture:
<img src="/images/p2viz/pic_name.png" alt="Screen" class="responsive">
-->
<h1 class="display-2">Weekly Updates on progress</h1>
<h1 style="font-size: 32pt">Week 1</h1>
<h2>Mirko</h2>
For the first week I started my work by designing the initial high-level Graphical User Interface with sidebar, tabs, colors. I then concentrated on the Array of Things APIs. I integrated useful functions that we would need in our project inspired by the official R wrapper APIs that are currently not working. I noticed that there were a few problems in the AoT APIs, and with the queries that they provide it is tricky to get the initial information that we need for the map (dataset with all the nodes and the sensor that each node provides). Since right now there is no trivial way (apparently) to get the sensors list from a node, I got the same information parsing the observations of each individual node, this process is slow due to the need of querying each node and slow response from the AoT servers, it takes approximately 30 seconds. Once I created the needed dataset I noticed some interesting problems: only 38 nodes currently responde with data observations (the toal number of nodes already deployed is 119). Also, I cannot explain why 2 nodes with different vsn (070 and 072) have same exact location and address. I then plotted on the map the locations of the sensors and created the selection layers for the users, where they can select to visualize only part a subset of sensors.
I saved the nodes information in FST format and created a variable that controls the update of the nodes information or retrieves the previously saved one. I also created a basic plot and the comparison for the Current data and all of the tracked measures coming from the AoT source: when clicking on a node the plot is displayed for the selected measures, the user can also select another node and compare it two the previous one by visualizing an identical plot below the most recent's one.
<h2>Abhishek</h2>
Started working on preprocessing the OpenAQ data. Task is to bring it to a certain dataframe format, so that reading it becomes easy.
<h2>Ashwani</h2>
During this week, I tried working with the R library for AoT and contacted the owner of repository for a fix for the recent changes.
We then moved forward with using the AoT API directly. We discussed about the various project requirements and how to implement them.
I worked on creating some of the graphical charts for the Array of Things data and added corresponding units of measure for them.
<h1 style="font-size: 32pt">Week 2</h1>
<h2>Mirko</h2>
If I thought the Array of Things data source was bad from the first week impressions, now I can confirm this is the worst APIs and service I've ever used. The challenges I faced this week were related to showing the last 24 hours and last 7 days of data in a line chart upon clicking on a sensor. I discovered that a lot of sensor that have data, randomly shut down and turn back on disrupting the service and making it difficult to understand what is going on with the data. I had trouble in finding a node which was pretty good in terms of providing data constantly, eventually I picked node 08D. The issue now was on how to query the API s to get the data I wanted. I initially tried to get all the observations in the last 24 hours and then preprocess them. I soon realized the infeasibility of this method because of the latency in downloading and preprocessing such a large quantity of data (around 100k rows) in real-time. It would take around 1 minute from when the user clicks the node to when the plot is generated. Way too much. I thought about a possible implementation in which data for each node are preloaded and processed when the application starts but that would take too much time and the user expects reactivity quickly. My final solution to solve this problem was that of subsampling the data (200 obs each hour/day) based on the timestamp. It also took me a long time to figure out how to create an HTTP request to the AoT APIs filtering based on timestamp. The official APIs documentation as of now lacks examples or proper explanation. Eventually I got it to work. Another cool surprised AoT gave us is that there are places in Chicago with -30 ppm of so2, which would be great if it was true but I'm afraid it's not really physically possible.
<h2>Abhishek</h2>
Done with the preprocessing script for openAQ data. Need to start working on creating 4 new tabs for displaying graph and tables for two subsets of data identified during our discussion.
<h2>Ashwani</h2>
During this week, I created the table for node and sensor information for Array of Things dataset. I then worked on the tabular visualzation for Darksky for the current time in a separte panel. The functionality to filter sites based on measures was added for the table representation of the sites. The graphical charts for Array of Things now can take input from the table and show corresponding visualizations. The selection of multiple inputs from table is a bit tricky and needs to be looked into in detail for the comparsion task.
<h1 style="font-size: 32pt">Alpha version</h1>
<h2>Mirko</h2>
During the third week and for the Alpha version I integrated OpenAQ nodes on map, created nodes legend in map, added 4 different map backgrounds and refreshing the charts every minute which completed all the C requirements.
<h2>Abhishek</h2>
Merged panels for openAQ and AoT pollutants which will be tab 1. Tab 2 will contain other variables like intensity, humidity, etc. Merge needs to be done for this also which will act as tab 2.
<h2>Ashwani</h2>
During this week, I fixed the map and table inputs issue for graphical plots to handle them independently. I preprocessed the darksky data and created the 24 hours and last 7 days charts for Darksky. I worked on the issue of comparison of plots where same nodes were being shown on comparsion. The comparison fix currently has been done for Array of Things map and table inputs and needs to be added for Darksky once the reorganization of charts is done.
<h1 style="font-size: 32pt">Week 4</h1>
<h2>Mirko</h2>
After the professor's approval on the new data source to be used as a graduate additional requirement, I integrated real-time traffic and congestion data taken from <a href="https://dev.socrata.com/foundry/data.cityofchicago.org/n4j6-wkkf">Chicago Traffic Tracker</a> in the application. In particular, the dataset comes with 5 important features, which are: start and end of longitude and latitude and current traffic speed. I decided to show thid data graphically on the map by creating colored lines with the following color convention.
colors: grey, no traffic (no cars), blue: normal/high speed and above, yellow: medium speed, orange: low speed, red: very low or zero speed.
By clicking on a street, a popup appears showing data such as: the name of the main street and the crossings it joins, numeric speed in mph, link to APIs.
The street layers can be toggled on or off from the respective layers menu.
<h2>Abhishek</h2>
Successfully merged AoT, openAQ and darksky into one panel. This would involve querying darksky and openaq simultaneously as parameters from both sites are displayed on the graph. I also added tables for displaying metrics and values for current node and also a comparison table for the previous node chosen.
<h2>Ashwani</h2>
During this week, I changed the sensor table presentation to make it visually better. I fixed the filtering of sites from sensor table to show sites based on measure availability. I added a button to hide the table based on user preference. I fixed various issues related to graphical plots for Darksky. I started working on the heatmaps and preprocessed the Array of Things API data required for its heatmap.
<h1 style="font-size: 32pt">Final delivery</h1>
<h2>Mirko</h2>
For the final week I adjusted all the sizes to make the application work well on the class wall, fixed some recurrent bugs. I also tried to decrease the loading time for the last 24 hours data for the Array of Things by getting all the entries in the last 24 hours, the advantage of this over the method in which I get samples of data for each hour is that only 1 request has to be made, and AoT has a slow response time so 1 request with a lot more data is faster than 24 requests with small data. The problem here was that if you pass as query parameters the instants defining the interval between now and 24 hours ago and don't modify the maximum size of the retrieved data, you will only get 200 results. If instead you modify the size, there is a limit of 100k in the API that is not enought for some nodes that could have more than 100k entries in the last 24 hours. In the end I decided to just switch back to the old and slow method. There is not much we can do to improve the speed of data gathering and plot generation for AoT since their APIs are the main bottleneck and they are also not working as expected.
<h2>Abhishek</h2>
Fixed various minor bugs. I changed colors for all variables used in the graph so that they all have colors which are easily differentiable. I also added documentation on the website related to the merged panel and also how the data is being displayed from the various sources into one panel.
<h2>Ashwani</h2>
During this week, I worked on completing the heatmaps for all the three data sources, Array of Things, Darksky and OpenAq.
I did preprocessing for all the three data source to get data for all the nodese for them. I created the various inputs for heatmaps for all the required measures. The heatmaps are created for various type of values min,max,average, for all time ranges current, last 24 hours and last 7 days. The randomization of Darksky key is done to select any key at random for a given set of keys. I fixed the slowness of previous click on the graphical plots. I explored the possibility of saving the previous data but it might causes some problems so the data is queried in real time which might affect performance depending on the response time of API and amount of data requested.
<!--
<h1 style="font-size: 32pt">Week 4</h1>
<h2>Mirko</h2>
No scrolling is possible in the application. Modified panels for map. Added confidence input in map and modified color opacity based on days with data in map. Minor fixes to colors and sizes for Yearly plots.
<h2>Abhishek</h2>
Completed the daily and totals for italy. Made several minor bug fixes. Added imperial conversion for daily AQI for both USA and italy. Implemented touch feature for ggvis plots.
<h2>Ashwani</h2>
Metric to imperial conversion for hourly plots for US. Worked on creation of hourly charts for one more country Italy.Added switch for
top 12 counties in US. Updated the hourly preprocessing script for italy for performance. Minor fixes for yearly plots.
-->
</div>
</div>
</section>
<style>
.marginauto {
margin: 10px auto 20px;
display: block;
}
</style>
<!-- services
================================================== -->
<!-- works
================================================== -->
<section id='video' class="s-works">
<div class="intro-wrap">
<div class="row section-header has-bottom-sep light-sep" data-aos="fade-up">
<div class="col-full">
<h3 class="subhead">Video</h3>
<h1 class="display-2 display-2--light">Video presentation</h1>
<!-- <iframe width="1080" height="720" src="https://www.youtube.com/embed/UCvc0h6yKMg" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>-->
<iframe width="1080" height="720" src="https://www.youtube.com/embed/XrqJ1vvaasA" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</div>
</div>
<div class="row works-content">
<div class="col-full masonry-wrap">
<div class="masonry">
</div>
</div>
</div>
</div>
</section>
<!-- contact
================================================== -->
<section id="contact" class="s-contact">
<div class="overlay"></div>
<div class="contact__line"></div>
<div class="row section-header" data-aos="fade-up">
<div class="col-full">
<h3 class="subhead">Contact Us</h3>
<h1 class="display-2 display-2--light">Reach out for a new project or just say hello</h1>
</div>
</div>
<div class="row contact-content" data-aos="fade-up">
<div class="contact-primary">
<h3 class="h6">Send Me A Message</h3>
<form name="contactForm" id="contactForm" method="post" action="" novalidate="novalidate">
<fieldset>
<div class="form-field">
<input name="contactName" type="text" id="contactName" placeholder="Your Name" value="" minlength="2" required="" aria-required="true" class="full-width">
</div>
<div class="form-field">
<input name="contactEmail" type="email" id="contactEmail" placeholder="Your Email" value="" required="" aria-required="true" class="full-width">
</div>
<div class="form-field">
<input name="contactSubject" type="text" id="contactSubject" placeholder="Subject" value="" class="full-width">
</div>
<div class="form-field">
<textarea name="contactMessage" id="contactMessage" placeholder="Your Message" rows="10" cols="50" required="" aria-required="true" class="full-width"></textarea>
</div>
<div class="form-field">
<button class="full-width btn--primary">Submit</button>
<div class="submit-loader">
<div class="text-loader">Sending...</div>
<div class="s-loader">
<div class="bounce1"></div>
<div class="bounce2"></div>
<div class="bounce3"></div>
</div>
</div>
</div>
</fieldset>
</form>
<!-- contact-warning -->
<div class="message-warning">
Something went wrong. Please try again.
</div>
<!-- contact-success -->
<div class="message-success">
Your message was sent, thank you!<br>
</div>
</div> <!-- end contact-primary -->
<div class="contact-secondary">
<div class="contact-info">
<h3 class="h6 hide-on-fullwidth">Contact Info</h3>
<div class="cinfo">
<h5>Where I live</h5>
<p>
Chicago, IL<br>
60622 USA
</p>
</div>
<div class="cinfo">
<h5>Email Me At</h5>
<p>
mrk23 at hotmail dot it<br>
</p>
</div>
<div class="cinfo">
<h5>Call Me At</h5>
<p>
Mobile: <br>
</p>
</div>
<ul class="contact-social">
<li>
<a href="https://www.facebook.com/mrkmantovani"><i class="fa fa-facebook" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://www.linkedin.com/in/mirkomantovani/"><i class="fa fa-linkedin" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://www.instagram.com/mirkomantovani/"><i class="fa fa-instagram" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://play.google.com/store/apps/developer?id=Mirko+Mantovani"><i class="fa fa-android" aria-hidden="true"></i></a>
</li>
<li>
<a href="https://github.com/mirkomantovani"><i class="fa fa-github" aria-hidden="true"></i></a>
</li>
</ul> <!-- end contact-social -->
</div> <!-- end contact-info -->
</div> <!-- end contact-secondary -->
</div> <!-- end contact-content -->
</section> <!-- end s-contact -->
<!-- footer
================================================== -->
<footer>
<div class="row footer-main">
<div class="col-six tab-full left footer-desc">
<div class="footer-logo"></div>
Portfolio of Mirko Mantovani
</div>
<div class="col-six tab-full right footer-subscribe">
<h4>Get Notified</h4>
<p></p>
<div class="subscribe-form">
<form id="mc-form" class="group" novalidate="true">
<input type="email" value="" name="EMAIL" class="email" id="mc-email" placeholder="Email Address" required="">
<input type="submit" name="subscribe" value="Subscribe">
<label for="mc-email" class="subscribe-message"></label>
</form>
</div>
</div>
</div> <!-- end footer-main -->
<div class="row footer-bottom">
<!--
<div class="col-twelve">
<div class="copyright">
<span>© Copyright Glint 2017</span>
<span>Site Template by <a href="">Colorlib</a></span>
</div>
<div class="go-top">
<a class="smoothscroll" title="Back to Top" href="#top"><i class="icon-arrow-up" aria-hidden="true"></i></a>
</div>
</div>
-->
</div> <!-- end footer-bottom -->
</footer> <!-- end footer -->
<!-- photoswipe background
================================================== -->
<div aria-hidden="true" class="pswp" role="dialog" tabindex="-1">
<div class="pswp__bg"></div>
<div class="pswp__scroll-wrap">
<div class="pswp__container">
<div class="pswp__item"></div>
<div class="pswp__item"></div>
<div class="pswp__item"></div>
</div>
<div class="pswp__ui pswp__ui--hidden">
<div class="pswp__top-bar">
<div class="pswp__counter"></div><button class="pswp__button pswp__button--close" title="Close (Esc)"></button> <button class="pswp__button pswp__button--share" title="Share"></button> <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button> <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
<div class="pswp__preloader">
<div class="pswp__preloader__icn">
<div class="pswp__preloader__cut">
<div class="pswp__preloader__donut"></div>
</div>
</div>
</div>
</div>
<div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
<div class="pswp__share-tooltip"></div>
</div><button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button> <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>
<div class="pswp__caption">
<div class="pswp__caption__center"></div>
</div>
</div>
</div>
</div> <!-- end photoSwipe background -->
<!-- preloader
================================================== -->
<div id="preloader">
<div id="loader">
<div class="line-scale-pulse-out">
<div></div>
<div></div>
<div></div>
<div></div>
<div></div>
</div>
</div>
</div>
<!-- Java Script
================================================== -->
<script src="/js/jquery-3.2.1.min.js"></script>
<script src="/js/plugins.js"></script>
<script src="/js/main.js"></script>
</body>
</html>