-
Notifications
You must be signed in to change notification settings - Fork 0
/
nohup.out
3733 lines (1996 loc) · 224 KB
/
nohup.out
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
start at mesos://master:5050
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
I0110 18:18:59.429769 29573 sched.cpp:226] Version: 1.0.1
I0110 18:18:59.432175 29571 sched.cpp:330] New master detected at [email protected]:5050
I0110 18:18:59.432433 29571 sched.cpp:341] No credentials provided. Attempting to register without authentication
I0110 18:18:59.433833 29571 sched.cpp:743] Framework registered with 944f520c-c667-4a69-a3b5-8b2032694724-0009
[Stage 21:=========> (2 + 10) / 12][Stage 21:===================> (4 + 8) / 12][Stage 21:======================================> (8 + 4) / 12][Stage 22:=================================> (7 + 5) / 12][Stage 22:==============================================> (10 + 2) / 12] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Exception in thread "main" org.apache.spark.SparkException: ArrayBuffer(java.nio.channels.ClosedChannelException, org.apache.spark.SparkException: Couldn't find leader offsets for Set([twitter,0]))
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.latestLeaderOffsets(DirectKafkaInputDStream.scala:133)
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.compute(DirectKafkaInputDStream.scala:158)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.MappedDStream.compute(MappedDStream.scala:36)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.ForEachDStream.generateJob(ForEachDStream.scala:48)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:117)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:116)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241)
at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104)
at org.apache.spark.streaming.DStreamGraph.generateJobs(DStreamGraph.scala:116)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:249)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:247)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.JobGenerator.generateJobs(JobGenerator.scala:247)
at org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:183)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:89)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:88)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
I0112 13:24:51.320494 5403 sched.cpp:1987] Asked to stop the driver
I0112 13:24:51.320674 29570 sched.cpp:1187] Stopping framework '944f520c-c667-4a69-a3b5-8b2032694724-0009'
Parameter error, please set local/master/mesos/mesos/cluster
will start at local
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
[Stage 201:============================> (2 + 2) / 4] [Stage 202:> (0 + 4) / 4][Stage 202:==============> (1 + 3) / 4][Stage 202:============================> (2 + 2) / 4][Stage 202:==========================================> (3 + 1) / 4] [Stage 204:==============> (1 + 3) / 4] [Stage 205:> (0 + 4) / 4][Stage 205:============================> (2 + 2) / 4][Stage 205:==========================================> (3 + 1) / 4] [Stage 208:==========================================> (3 + 1) / 4] [Stage 210:==============> (1 + 3) / 4][Stage 210:==========================================> (3 + 1) / 4] [Stage 211:> (0 + 4) / 4][Stage 211:==============> (1 + 3) / 4][Stage 211:============================> (2 + 2) / 4] [Stage 213:==============> (1 + 3) / 4][Stage 213:============================> (2 + 2) / 4] [Stage 214:> (0 + 4) / 4][Stage 214:==============> (1 + 3) / 4][Stage 214:============================> (2 + 2) / 4][Stage 214:==========================================> (3 + 1) / 4] [Stage 216:==============> (1 + 3) / 4][Stage 216:==========================================> (3 + 1) / 4] [Stage 217:> (0 + 4) / 4][Stage 217:============================> (2 + 2) / 4][Stage 217:==========================================> (3 + 1) / 4] [Stage 219:==========================================> (3 + 1) / 4] [Stage 220:============================> (2 + 2) / 4][Stage 220:==========================================> (3 + 1) / 4] [Stage 222:==========================================> (3 + 1) / 4] [Stage 223:==============> (1 + 3) / 4][Stage 223:============================> (2 + 2) / 4] [Stage 225:==============> (1 + 3) / 4][Stage 225:==========================================> (3 + 1) / 4] [Stage 226:> (0 + 4) / 4][Stage 226:==============> (1 + 3) / 4][Stage 226:==========================================> (3 + 1) / 4] [Stage 228:==============> (1 + 3) / 4] [Stage 229:> (0 + 4) / 4][Stage 229:==============> (1 + 3) / 4][Stage 229:============================> (2 + 2) / 4][Stage 229:==========================================> (3 + 1) / 4] [Stage 231:==========================================> (3 + 1) / 4] [Stage 232:============================> (2 + 2) / 4][Stage 232:==========================================> (3 + 1) / 4] [Stage 234:==============> (1 + 3) / 4][Stage 234:==========================================> (3 + 1) / 4] [Stage 235:> (0 + 4) / 4][Stage 235:==========================================> (3 + 1) / 4] [Stage 237:==============> (1 + 3) / 4][Stage 237:============================> (2 + 2) / 4][Stage 237:==========================================> (3 + 1) / 4] [Stage 238:> (0 + 4) / 4][Stage 238:==============> (1 + 3) / 4][Stage 238:============================> (2 + 2) / 4] [Stage 240:============================> (2 + 2) / 4] [Stage 241:==============> (1 + 3) / 4][Stage 241:============================> (2 + 2) / 4] [Stage 243:==============> (1 + 3) / 4] [Stage 244:> (0 + 4) / 4][Stage 244:==============> (1 + 3) / 4][Stage 244:==========================================> (3 + 1) / 4] [Stage 246:============================> (2 + 2) / 4] [Stage 247:==============> (1 + 3) / 4][Stage 247:============================> (2 + 2) / 4][Stage 247:==========================================> (3 + 1) / 4] [Stage 249:==============> (1 + 3) / 4][Stage 249:==========================================> (3 + 1) / 4] [Stage 250:> (0 + 4) / 4][Stage 250:==============> (1 + 3) / 4][Stage 250:============================> (2 + 2) / 4] [Stage 252:==============> (1 + 3) / 4] [Stage 253:> (0 + 4) / 4][Stage 253:==============> (1 + 3) / 4][Stage 253:==========================================> (3 + 1) / 4] [Stage 255:============================> (2 + 2) / 4] [Stage 256:> (0 + 4) / 4][Stage 256:============================> (2 + 2) / 4] [Stage 258:==============> (1 + 3) / 4][Stage 258:============================> (2 + 2) / 4] [Stage 259:> (0 + 4) / 4][Stage 259:==============> (1 + 3) / 4][Stage 259:============================> (2 + 2) / 4] [Stage 261:==============> (1 + 3) / 4][Stage 261:============================> (2 + 2) / 4][Stage 261:==========================================> (3 + 1) / 4] [Stage 262:> (0 + 4) / 4][Stage 262:==============> (1 + 3) / 4][Stage 262:============================> (2 + 2) / 4][Stage 262:==========================================> (3 + 1) / 4] [Stage 264:============================> (2 + 2) / 4] [Stage 265:==============> (1 + 3) / 4][Stage 265:============================> (2 + 2) / 4][Stage 265:==========================================> (3 + 1) / 4] [Stage 267:==============> (1 + 3) / 4][Stage 267:==========================================> (3 + 1) / 4] [Stage 268:> (0 + 4) / 4][Stage 268:==============> (1 + 3) / 4][Stage 268:==========================================> (3 + 1) / 4] [Stage 270:============================> (2 + 2) / 4][Stage 270:==========================================> (3 + 1) / 4] [Stage 271:==============> (1 + 3) / 4][Stage 271:============================> (2 + 2) / 4][Stage 271:==========================================> (3 + 1) / 4] [Stage 273:==============> (1 + 3) / 4] [Stage 274:> (0 + 4) / 4][Stage 274:==============> (1 + 3) / 4][Stage 274:============================> (2 + 2) / 4][Stage 274:==========================================> (3 + 1) / 4] [Stage 276:==========================================> (3 + 1) / 4] [Stage 277:============================> (2 + 2) / 4][Stage 277:==========================================> (3 + 1) / 4] [Stage 279:==============> (1 + 3) / 4][Stage 279:============================> (2 + 2) / 4][Stage 279:==========================================> (3 + 1) / 4] [Stage 280:> (0 + 4) / 4][Stage 280:==============> (1 + 3) / 4] [Stage 282:==========================================> (3 + 1) / 4] [Stage 283:==============> (1 + 3) / 4][Stage 283:============================> (2 + 2) / 4] [Stage 285:==============> (1 + 3) / 4][Stage 285:==========================================> (3 + 1) / 4] [Stage 286:> (0 + 4) / 4][Stage 286:==============> (1 + 3) / 4][Stage 286:============================> (2 + 2) / 4] [Stage 288:==========================================> (3 + 1) / 4] [Stage 289:============================> (2 + 2) / 4][Stage 289:==========================================> (3 + 1) / 4] [Stage 291:==============> (1 + 3) / 4][Stage 291:============================> (2 + 2) / 4][Stage 291:==========================================> (3 + 1) / 4] [Stage 292:> (0 + 4) / 4][Stage 292:==============> (1 + 3) / 4][Stage 292:============================> (2 + 2) / 4] [Stage 294:============================> (2 + 2) / 4][Stage 294:==========================================> (3 + 1) / 4] [Stage 295:==============> (1 + 3) / 4][Stage 295:============================> (2 + 2) / 4][Stage 295:==========================================> (3 + 1) / 4] [Stage 297:==============> (1 + 3) / 4][Stage 297:============================> (2 + 2) / 4][Stage 297:==========================================> (3 + 1) / 4] [Stage 298:> (0 + 4) / 4][Stage 298:==============> (1 + 3) / 4][Stage 298:============================> (2 + 2) / 4] [Stage 300:============================> (2 + 2) / 4][Stage 300:==========================================> (3 + 1) / 4] [Stage 301:==============> (1 + 3) / 4][Stage 301:============================> (2 + 2) / 4][Stage 301:==========================================> (3 + 1) / 4] [Stage 303:============================> (2 + 2) / 4][Stage 303:==========================================> (3 + 1) / 4] [Stage 304:==============> (1 + 3) / 4][Stage 304:============================> (2 + 2) / 4] [Stage 306:============================> (2 + 2) / 4][Stage 306:==========================================> (3 + 1) / 4] [Stage 307:==============> (1 + 3) / 4][Stage 307:============================> (2 + 2) / 4] [Stage 310:============================> (2 + 2) / 4][Stage 310:==========================================> (3 + 1) / 4] [Stage 311:==============> (1 + 3) / 4][Stage 311:============================> (2 + 2) / 4][Stage 311:==========================================> (3 + 1) / 4] [Stage 313:============================> (2 + 2) / 4] [Stage 314:==============> (1 + 3) / 4][Stage 314:============================> (2 + 2) / 4] [Stage 316:============================> (2 + 2) / 4] [Stage 317:==============> (1 + 3) / 4][Stage 317:============================> (2 + 2) / 4][Stage 317:==========================================> (3 + 1) / 4] [Stage 319:============================> (2 + 2) / 4][Stage 319:==========================================> (3 + 1) / 4] [Stage 320:> (0 + 4) / 4][Stage 320:==============> (1 + 3) / 4][Stage 320:==========================================> (3 + 1) / 4] [Stage 322:==============> (1 + 3) / 4][Stage 322:==========================================> (3 + 1) / 4] [Stage 323:> (0 + 4) / 4][Stage 323:==============> (1 + 3) / 4][Stage 323:============================> (2 + 2) / 4] [Stage 325:==============> (1 + 3) / 4][Stage 325:============================> (2 + 2) / 4][Stage 325:==========================================> (3 + 1) / 4] [Stage 326:> (0 + 4) / 4][Stage 326:==============> (1 + 3) / 4][Stage 326:============================> (2 + 2) / 4][Stage 326:==========================================> (3 + 1) / 4] [Stage 330:============================> (2 + 2) / 4] [Stage 331:==============> (1 + 3) / 4][Stage 331:============================> (2 + 2) / 4][Stage 331:==========================================> (3 + 1) / 4] [Stage 333:============================> (2 + 2) / 4][Stage 333:==========================================> (3 + 1) / 4] [Stage 334:==============> (1 + 3) / 4][Stage 334:============================> (2 + 2) / 4] [Stage 336:==========================================> (3 + 1) / 4] [Stage 337:============================> (2 + 2) / 4][Stage 337:==========================================> (3 + 1) / 4] [Stage 339:============================> (2 + 2) / 4] [Stage 340:==============> (1 + 3) / 4][Stage 340:============================> (2 + 2) / 4] [Stage 342:==========================================> (3 + 1) / 4] [Stage 343:============================> (2 + 2) / 4][Stage 343:==========================================> (3 + 1) / 4] [Stage 345:==============> (1 + 3) / 4] [Stage 346:> (0 + 4) / 4][Stage 346:==============> (1 + 3) / 4][Stage 346:==========================================> (3 + 1) / 4] [Stage 348:============================> (2 + 2) / 4] [Stage 349:> (0 + 4) / 4][Stage 349:==============> (1 + 3) / 4][Stage 349:==========================================> (3 + 1) / 4] [Stage 351:==============> (1 + 3) / 4] [Stage 352:> (0 + 4) / 4][Stage 352:==============> (1 + 3) / 4][Stage 352:============================> (2 + 2) / 4] [Stage 354:==========================================> (3 + 1) / 4] [Stage 355:==========================================> (3 + 1) / 4] [Stage 357:==============> (1 + 3) / 4][Stage 357:============================> (2 + 2) / 4] [Stage 358:> (0 + 4) / 4][Stage 358:==============> (1 + 3) / 4][Stage 358:==========================================> (3 + 1) / 4] [Stage 360:==============> (1 + 3) / 4] [Stage 361:> (0 + 4) / 4][Stage 361:==============> (1 + 3) / 4][Stage 361:==========================================> (3 + 1) / 4] [Stage 363:==============> (1 + 3) / 4] [Stage 364:> (0 + 4) / 4][Stage 364:==============> (1 + 3) / 4][Stage 364:==========================================> (3 + 1) / 4] [Stage 366:==============> (1 + 3) / 4][Stage 366:==========================================> (3 + 1) / 4] [Stage 367:> (0 + 4) / 4][Stage 367:==============> (1 + 3) / 4][Stage 367:============================> (2 + 2) / 4] [Stage 369:> (0 + 4) / 4][Stage 369:==============> (1 + 3) / 4][Stage 369:============================> (2 + 2) / 4][Stage 369:==========================================> (3 + 1) / 4] [Stage 370:> (0 + 4) / 4][Stage 370:==============> (1 + 3) / 4][Stage 370:============================> (2 + 2) / 4][Stage 370:==========================================> (3 + 1) / 4] [Stage 372:============================> (2 + 2) / 4] [Stage 373:==============> (1 + 3) / 4][Stage 373:============================> (2 + 2) / 4][Stage 373:==========================================> (3 + 1) / 4] [Stage 375:==============> (1 + 3) / 4][Stage 375:============================> (2 + 2) / 4] [Stage 376:> (0 + 4) / 4][Stage 376:==============> (1 + 3) / 4][Stage 376:============================> (2 + 2) / 4] [Stage 378:==============> (1 + 3) / 4][Stage 378:============================> (2 + 2) / 4][Stage 378:==========================================> (3 + 1) / 4] [Stage 379:> (0 + 4) / 4][Stage 379:==============> (1 + 3) / 4][Stage 379:============================> (2 + 2) / 4][Stage 379:==========================================> (3 + 1) / 4] [Stage 381:==============> (1 + 3) / 4][Stage 381:============================> (2 + 2) / 4][Stage 381:==========================================> (3 + 1) / 4] [Stage 382:> (0 + 4) / 4][Stage 382:==============> (1 + 3) / 4][Stage 382:==========================================> (3 + 1) / 4] [Stage 384:============================> (2 + 2) / 4][Stage 384:==========================================> (3 + 1) / 4] [Stage 385:==============> (1 + 3) / 4][Stage 385:============================> (2 + 2) / 4] [Stage 387:==============> (1 + 3) / 4][Stage 387:============================> (2 + 2) / 4][Stage 387:==========================================> (3 + 1) / 4] [Stage 388:> (0 + 4) / 4][Stage 388:==========================================> (3 + 1) / 4] [Stage 390:==============> (1 + 3) / 4][Stage 390:==========================================> (3 + 1) / 4] [Stage 391:> (0 + 4) / 4][Stage 391:==============> (1 + 3) / 4][Stage 391:==========================================> (3 + 1) / 4] [Stage 393:==========================================> (3 + 1) / 4] [Stage 394:============================> (2 + 2) / 4][Stage 394:==========================================> (3 + 1) / 4] [Stage 396:==============> (1 + 3) / 4][Stage 396:============================> (2 + 2) / 4][Stage 396:==========================================> (3 + 1) / 4] [Stage 397:> (0 + 4) / 4][Stage 397:==============> (1 + 3) / 4][Stage 397:============================> (2 + 2) / 4][Stage 397:==========================================> (3 + 1) / 4] [Stage 399:==============> (1 + 3) / 4][Stage 399:============================> (2 + 2) / 4][Stage 399:==========================================> (3 + 1) / 4] [Stage 400:> (0 + 4) / 4][Stage 400:==============> (1 + 3) / 4][Stage 400:============================> (2 + 2) / 4] [Stage 402:==============> (1 + 3) / 4][Stage 402:==========================================> (3 + 1) / 4] [Stage 403:> (0 + 4) / 4][Stage 403:============================> (2 + 2) / 4][Stage 403:==========================================> (3 + 1) / 4] [Stage 405:> (0 + 4) / 4][Stage 405:==============> (1 + 3) / 4][Stage 405:============================> (2 + 2) / 4] [Stage 406:> (0 + 4) / 4][Stage 406:==============> (1 + 3) / 4][Stage 406:============================> (2 + 2) / 4][Stage 406:==========================================> (3 + 1) / 4] [Stage 410:> (0 + 4) / 4][Stage 410:============================> (2 + 2) / 4][Stage 410:==========================================> (3 + 1) / 4] [Stage 411:> (0 + 4) / 4][Stage 411:==============> (1 + 3) / 4][Stage 411:==========================================> (3 + 1) / 4] [Stage 415:==============> (1 + 3) / 4][Stage 415:============================> (2 + 2) / 4][Stage 415:==========================================> (3 + 1) / 4] [Stage 416:> (0 + 4) / 4][Stage 416:============================> (2 + 2) / 4] [Stage 418:==============> (1 + 3) / 4][Stage 418:============================> (2 + 2) / 4] [Stage 419:> (0 + 4) / 4][Stage 419:==============> (1 + 3) / 4][Stage 419:============================> (2 + 2) / 4] [Stage 421:============================> (2 + 2) / 4] [Stage 422:> (0 + 4) / 4][Stage 422:==============> (1 + 3) / 4][Stage 422:==========================================> (3 + 1) / 4] [Stage 424:==============> (1 + 3) / 4] [Stage 425:> (0 + 4) / 4][Stage 425:==============> (1 + 3) / 4] [Stage 427:==============> (1 + 3) / 4] [Stage 428:> (0 + 4) / 4][Stage 428:==============> (1 + 3) / 4][Stage 428:============================> (2 + 2) / 4][Stage 428:==========================================> (3 + 1) / 4] [Stage 430:============================> (2 + 2) / 4] [Stage 431:> (0 + 4) / 4][Stage 431:==============> (1 + 3) / 4][Stage 431:==========================================> (3 + 1) / 4] [Stage 433:> (0 + 4) / 4][Stage 433:==========================================> (3 + 1) / 4] [Stage 434:> (0 + 4) / 4][Stage 434:==============> (1 + 3) / 4][Stage 434:============================> (2 + 2) / 4] [Stage 436:============================> (2 + 2) / 4] [Stage 437:> (0 + 4) / 4][Stage 437:==============> (1 + 3) / 4][Stage 437:============================> (2 + 2) / 4] [Stage 439:==============> (1 + 3) / 4][Stage 439:============================> (2 + 2) / 4][Stage 439:==========================================> (3 + 1) / 4] [Stage 440:> (0 + 4) / 4][Stage 440:==============> (1 + 3) / 4][Stage 440:============================> (2 + 2) / 4][Stage 440:==========================================> (3 + 1) / 4] [Stage 442:============================> (2 + 2) / 4][Stage 442:==========================================> (3 + 1) / 4] [Stage 443:> (0 + 4) / 4][Stage 443:============================> (2 + 2) / 4][Stage 443:==========================================> (3 + 1) / 4] [Stage 445:==============> (1 + 3) / 4][Stage 445:============================> (2 + 2) / 4][Stage 445:==========================================> (3 + 1) / 4] [Stage 446:> (0 + 4) / 4][Stage 446:==============> (1 + 3) / 4][Stage 446:============================> (2 + 2) / 4][Stage 446:==========================================> (3 + 1) / 4] [Stage 448:==============> (1 + 3) / 4][Stage 448:==========================================> (3 + 1) / 4] [Stage 449:> (0 + 4) / 4][Stage 449:==============> (1 + 3) / 4][Stage 449:============================> (2 + 2) / 4][Stage 449:==========================================> (3 + 1) / 4] [Stage 451:==========================================> (3 + 1) / 4] [Stage 452:==============> (1 + 3) / 4][Stage 452:============================> (2 + 2) / 4][Stage 452:==========================================> (3 + 1) / 4] [Stage 454:============================> (2 + 2) / 4][Stage 454:==========================================> (3 + 1) / 4] Exception in thread "streaming-job-executor-0" java.lang.Error: java.lang.InterruptedException
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1148)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply$mcV$sp(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2765)
at org.apache.spark.sql.Dataset.foreachPartition(Dataset.scala:2304)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:273)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:260)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
... 2 more
start at mesos://master:5050
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
I0112 15:59:12.984256 14620 sched.cpp:226] Version: 1.0.1
I0112 15:59:12.986302 14614 sched.cpp:330] New master detected at [email protected]:5050
I0112 15:59:12.986647 14614 sched.cpp:341] No credentials provided. Attempting to register without authentication
I0112 15:59:12.988641 14614 sched.cpp:743] Framework registered with 2a3391c2-26b9-4594-8be7-f50d3c0fafd7-0003
[Stage 21:=========> (2 + 10) / 12][Stage 21:============================> (6 + 6) / 12][Stage 21:==============================================> (10 + 2) / 12] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
[Stage 28:> (0 + 3) / 4][Stage 28:> (0 + 4) / 4][Stage 28:==============> (1 + 3) / 4][Stage 28:=============================> (2 + 2) / 4][Stage 28:===========================================> (3 + 1) / 4] [Stage 29:> (0 + 3) / 4][Stage 29:> (0 + 4) / 4][Stage 29:==============> (1 + 3) / 4][Stage 29:=============================> (2 + 2) / 4][Stage 29:===========================================> (3 + 1) / 4] [Stage 31:======================================> (2 + 0) / 3] [Stage 33:======================================> (2 + 0) / 3] [Stage 34:===========================================> (3 + 0) / 4] [Stage 35:=============================> (2 + 1) / 4][Stage 35:===========================================> (3 + 0) / 4] [Stage 37:======================================> (2 + 0) / 3] [Stage 39:======================================> (2 + 0) / 3] [Stage 41:======================================> (2 + 0) / 3] [Stage 43:======================================> (2 + 0) / 3][Stage 43:======================================> (2 + 1) / 3] [Stage 44:===========================================> (3 + 0) / 4][Stage 44:===========================================> (3 + 1) / 4] [Stage 45:===========================================> (3 + 0) / 4][Stage 45:===========================================> (3 + 1) / 4] [Stage 47:======================================> (2 + 0) / 3] [Stage 48:===========================================> (3 + 0) / 4] [Stage 49:=============================> (2 + 1) / 4][Stage 49:===========================================> (3 + 0) / 4] [Stage 51:===========================================> (3 + 0) / 4] [Stage 52:=============================> (2 + 1) / 4][Stage 52:===========================================> (3 + 0) / 4] [Stage 54:======================================> (2 + 0) / 3] [Stage 56:======================================> (2 + 0) / 3] [Stage 58:======================================> (2 + 0) / 3] [Stage 59:===========================================> (3 + 0) / 4] [Stage 60:=============================> (2 + 1) / 4][Stage 60:===========================================> (3 + 0) / 4] [Stage 62:======================================> (2 + 0) / 3] [Stage 64:======================================> (2 + 0) / 3] [Stage 66:======================================> (2 + 0) / 3] [Stage 67:===========================================> (3 + 0) / 4] [Stage 68:=============================> (2 + 1) / 4][Stage 68:===========================================> (3 + 0) / 4] [Stage 70:======================================> (2 + 0) / 3] [Stage 71:===========================================> (3 + 0) / 4][Stage 71:===========================================> (3 + 1) / 4] [Stage 72:===========================================> (3 + 0) / 4][Stage 72:===========================================> (3 + 1) / 4] [Stage 74:======================================> (2 + 0) / 3] [Stage 76:======================================> (2 + 0) / 3] [Stage 78:===========================================> (3 + 0) / 4][Stage 78:===========================================> (3 + 1) / 4] [Stage 79:=============================> (2 + 1) / 4][Stage 79:===========================================> (3 + 0) / 4][Stage 79:===========================================> (3 + 1) / 4] [Stage 81:======================================> (2 + 0) / 3] [Stage 83:======================================> (2 + 0) / 3][Stage 83:======================================> (2 + 1) / 3] [Stage 85:======================================> (2 + 0) / 3] [Stage 87:======================================> (2 + 0) / 3] [Stage 88:===========================================> (3 + 0) / 4] [Stage 89:==============> (1 + 2) / 4][Stage 89:===========================================> (3 + 0) / 4] [Stage 91:======================================> (2 + 0) / 3][Stage 91:======================================> (2 + 1) / 3] [Stage 92:===========================================> (3 + 0) / 4][Stage 92:===========================================> (3 + 1) / 4] [Stage 93:===========================================> (3 + 0) / 4][Stage 93:===========================================> (3 + 1) / 4] [Stage 95:======================================> (2 + 0) / 3] [Stage 97:======================================> (2 + 0) / 3] [Stage 99:======================================> (2 + 0) / 3] [Stage 101:======================================> (2 + 0) / 3] [Stage 103:======================================> (2 + 0) / 3] [Stage 105:======================================> (2 + 0) / 3] [Stage 106:==========================================> (3 + 0) / 4] [Stage 107:============================> (2 + 1) / 4][Stage 107:==========================================> (3 + 0) / 4] [Stage 109:======================================> (2 + 0) / 3] [Stage 111:======================================> (2 + 0) / 3] [Stage 113:==========================================> (3 + 0) / 4] [Stage 114:============================> (2 + 1) / 4][Stage 114:==========================================> (3 + 0) / 4] [Stage 116:======================================> (2 + 0) / 3] [Stage 118:======================================> (2 + 0) / 3] [Stage 119:==========================================> (3 + 0) / 4] [Stage 120:============================> (2 + 1) / 4][Stage 120:==========================================> (3 + 0) / 4] [Stage 122:======================================> (2 + 0) / 3] [Stage 124:======================================> (2 + 0) / 3] [Stage 125:==========================================> (3 + 0) / 4]Exception in thread "streaming-job-executor-0" java.lang.Error: java.lang.InterruptedException
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1148)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1981)
at org.apache.spark.rdd.RDD$$anonfun$fold$1.apply(RDD.scala:1088)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.fold(RDD.scala:1082)
at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:69)
at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:329)
at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:329)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:328)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:262)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:260)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
... 2 more
I0112 16:05:50.712224 15446 sched.cpp:1987] Asked to stop the driver
I0112 16:05:50.712430 14614 sched.cpp:1187] Stopping framework '2a3391c2-26b9-4594-8be7-f50d3c0fafd7-0003'
start at spark://master:7077
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 21:==================> (12 + 24) / 36][Stage 21:==============================> (20 + 16) / 36] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 27:> (0 + 10) / 10][Stage 27:===========> (2 + 8) / 10][Stage 27:==================================> (6 + 4) / 10][Stage 27:=============================================> (8 + 2) / 10] [Stage 28:> (0 + 10) / 10][Stage 28:===========> (2 + 8) / 10][Stage 28:=================> (3 + 7) / 10][Stage 28:======================> (4 + 6) / 10][Stage 28:============================> (5 + 5) / 10][Stage 28:==================================> (6 + 4) / 10][Stage 28:=======================================> (7 + 3) / 10][Stage 28:=============================================> (8 + 2) / 10][Stage 28:===================================================> (9 + 1) / 10] [Stage 217:> (0 + 0) / 10] [Stage 299:> (0 + 0) / 10] [Stage 713:> (0 + 0) / 10] [Stage 717:> (0 + 0) / 10] [Stage 719:> (0 + 0) / 10] [Stage 725:> (0 + 0) / 10] [Stage 727:> (0 + 0) / 10] [Stage 729:> (0 + 0) / 10] [Stage 731:> (0 + 0) / 10] Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 7 in stage 888.0 failed 4 times, most recent failure: Lost task 7.3 in stage 888.0 (TID 8861, 192.168.4.71, executor 3): org.apache.spark.SparkException: Couldn't connect to leader for topic tweet 4: java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator$$anonfun$connectLeader$1.apply(KafkaRDD.scala:168)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator$$anonfun$connectLeader$1.apply(KafkaRDD.scala:168)
at scala.util.Either.fold(Either.scala:98)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator.connectLeader(KafkaRDD.scala:167)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator.<init>(KafkaRDD.scala:159)
at org.apache.spark.streaming.kafka.KafkaRDD.compute(KafkaRDD.scala:136)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply$mcV$sp(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2765)
at org.apache.spark.sql.Dataset.foreachPartition(Dataset.scala:2304)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:258)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:253)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.spark.SparkException: Couldn't connect to leader for topic tweet 4: java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator$$anonfun$connectLeader$1.apply(KafkaRDD.scala:168)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator$$anonfun$connectLeader$1.apply(KafkaRDD.scala:168)
at scala.util.Either.fold(Either.scala:98)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator.connectLeader(KafkaRDD.scala:167)
at org.apache.spark.streaming.kafka.KafkaRDD$KafkaRDDIterator.<init>(KafkaRDD.scala:159)
at org.apache.spark.streaming.kafka.KafkaRDD.compute(KafkaRDD.scala:136)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:99)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
... 3 more
start at spark://master:7077
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 21:============> (8 + 28) / 36][Stage 21:==================> (12 + 24) / 36][Stage 21:===========================================> (28 + 8) / 36] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 28:======================> (4 + 6) / 10] [Stage 315:> (0 + 0) / 10] [Stage 327:> (0 + 0) / 10] [Stage 333:> (0 + 0) / 10] [Stage 737:> (0 + 0) / 10] [Stage 739:> (0 + 0) / 10] [Stage 1161:> (0 + 0) / 10] [Stage 1567:> (0 + 0) / 10] [Stage 1888:============================================> (8 + 2) / 10] [Stage 1969:> (0 + 0) / 10] [Stage 1979:> (0 + 0) / 10] [Stage 1981:> (0 + 0) / 10] [Stage 2017:> (0 + 0) / 10] [Stage 2377:> (0 + 0) / 10] [Stage 2399:> (0 + 0) / 10] [Stage 2592:============================================> (8 + 2) / 10] [Stage 2815:> (0 + 0) / 10] [Stage 3223:> (0 + 0) / 10] [Stage 3572:============================================> (8 + 2) / 10][Stage 3572:=================================================> (9 + 1) / 10] [Stage 3814:============================================> (8 + 2) / 10] [Stage 3815:============================================> (8 + 0) / 10][Stage 3815:============================================> (8 + 2) / 10] [Stage 3843:> (0 + 0) / 10] [Stage 3847:> (0 + 0) / 10] [Stage 3855:> (0 + 0) / 10] [Stage 3871:> (0 + 0) / 10] [Stage 3874:=================================================> (9 + 1) / 10]Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 8 in stage 3874.0 failed 4 times, most recent failure: Lost task 8.3 in stage 3874.0 (TID 38723, 192.168.4.73, executor 0): ExecutorLostFailure (executor 0 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply$mcV$sp(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2765)
at org.apache.spark.sql.Dataset.foreachPartition(Dataset.scala:2304)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:257)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:252)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
[Stage 3875:============================================> (8 + 2) / 10]Exception in thread "streaming-job-executor-0" java.lang.Error: java.lang.InterruptedException
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1148)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1981)
at org.apache.spark.rdd.RDD$$anonfun$fold$1.apply(RDD.scala:1088)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.fold(RDD.scala:1082)
at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:69)
at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:329)
at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:329)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:328)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:254)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:252)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
... 2 more
Parameter error, please set local/master/mesos/mesos/cluster
will start at local
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 28:===================================================> (9 + 1) / 10] [Stage 30:=============================================> (8 + 2) / 10] [Stage 32:=============================================> (8 + 2) / 10] [Stage 36:=============================================> (8 + 2) / 10] [Stage 38:=============================================> (8 + 2) / 10] start at spark://master:7077
[Stage 42:===================================================> (9 + 1) / 10] Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 46:=============================================> (8 + 2) / 10] [Stage 50:===================================================> (9 + 1) / 10] [Stage 52:===================================================> (9 + 1) / 10] [Stage 64:=============================================> (8 + 2) / 10] [Stage 70:===================================================> (9 + 1) / 10] [Stage 80:=============================================> (8 + 2) / 10] [Stage 82:=============================================> (8 + 2) / 10] [Stage 90:=============================================> (8 + 2) / 10] [Stage 92:=============================================> (8 + 2) / 10] [Stage 99:=============================================> (8 + 2) / 10] [Stage 102:> (0 + 0) / 10] [Stage 103:==================================================> (9 + 1) / 10] [Stage 105:==================================================> (9 + 1) / 10] [Stage 109:============================================> (8 + 2) / 10] [Stage 114:> (0 + 0) / 10] [Stage 115:============================================> (8 + 2) / 10] [Stage 117:============================================> (8 + 2) / 10] Exception in thread "main" org.apache.spark.SparkException: Job 7 cancelled because SparkContext was shut down
at org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:808)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:806)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:78)
at org.apache.spark.scheduler.DAGScheduler.cleanUpAfterSchedulerStop(DAGScheduler.scala:806)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onStop(DAGScheduler.scala:1668)
at org.apache.spark.util.EventLoop.stop(EventLoop.scala:83)
at org.apache.spark.scheduler.DAGScheduler.stop(DAGScheduler.scala:1587)
at org.apache.spark.SparkContext$$anonfun$stop$8.apply$mcV$sp(SparkContext.scala:1826)
at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1283)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1825)
at org.apache.spark.SparkContext$$anonfun$2.apply$mcV$sp(SparkContext.scala:581)
at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:216)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:188)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1951)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:188)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.rdd.RDD$$anonfun$take$1.apply(RDD.scala:1353)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.take(RDD.scala:1326)
at org.apache.spark.rdd.RDD$$anonfun$first$1.apply(RDD.scala:1367)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.first(RDD.scala:1366)
at org.apache.spark.ml.util.DefaultParamsReader$.loadMetadata(ReadWrite.scala:379)
at org.apache.spark.ml.util.DefaultParamsReader$.loadParamsInstance(ReadWrite.scala:433)
at org.apache.spark.ml.Pipeline$SharedReadWrite$$anonfun$4.apply(Pipeline.scala:273)
at org.apache.spark.ml.Pipeline$SharedReadWrite$$anonfun$4.apply(Pipeline.scala:271)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:186)
at org.apache.spark.ml.Pipeline$SharedReadWrite$.load(Pipeline.scala:271)
at org.apache.spark.ml.Pipeline$PipelineReader.load(Pipeline.scala:214)
at org.apache.spark.ml.Pipeline$PipelineReader.load(Pipeline.scala:208)
at org.apache.spark.ml.util.DefaultParamsReader$.loadParamsInstance(ReadWrite.scala:435)
at org.apache.spark.ml.tuning.ValidatorParams$.loadImpl(ValidatorParams.scala:167)
at org.apache.spark.ml.tuning.CrossValidatorModel$CrossValidatorModelReader.load(CrossValidator.scala:271)
at org.apache.spark.ml.tuning.CrossValidatorModel$CrossValidatorModelReader.load(CrossValidator.scala:262)
at org.apache.spark.ml.util.MLReadable$class.load(ReadWrite.scala:215)
at org.apache.spark.ml.tuning.CrossValidatorModel$.load(CrossValidator.scala:246)
at SparkCore$.main(SparkCore.scala:208)
at SparkCore.main(SparkCore.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
start at spark://master:7077
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 21:==============> (8 + 24) / 32] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 27:=============================================> (8 + 0) / 10][Stage 27:===================================================> (9 + 1) / 10] [Stage 28:===========> (2 + 6) / 10][Stage 28:=============================================> (8 + 0) / 10][Stage 28:=============================================> (8 + 2) / 10] [Stage 29:=============================================> (8 + 0) / 10] [Stage 30:=============================================> (8 + 0) / 10][Stage 30:===================================================> (9 + 1) / 10] [Stage 31:=============================================> (8 + 0) / 10] [Stage 32:=============================================> (8 + 0) / 10][Stage 32:=============================================> (8 + 2) / 10] [Stage 33:=============================================> (8 + 0) / 10] [Stage 34:=============================================> (8 + 0) / 10][Stage 34:===================================================> (9 + 1) / 10] [Stage 35:=============================================> (8 + 0) / 10] [Stage 36:=============================================> (8 + 0) / 10][Stage 36:=============================================> (8 + 2) / 10] [Stage 37:=============================================> (8 + 0) / 10] [Stage 38:=============================================> (8 + 0) / 10] [Stage 39:=============================================> (8 + 0) / 10] [Stage 40:=============================================> (8 + 0) / 10][Stage 40:=============================================> (8 + 2) / 10] [Stage 41:=============================================> (8 + 0) / 10] [Stage 42:=============================================> (8 + 0) / 10][Stage 42:=============================================> (8 + 2) / 10] [Stage 43:=============================================> (8 + 0) / 10] [Stage 44:=============================================> (8 + 0) / 10] [Stage 45:=============================================> (8 + 0) / 10] [Stage 46:=============================================> (8 + 0) / 10][Stage 46:=============================================> (8 + 2) / 10] [Stage 47:=============================================> (8 + 0) / 10] [Stage 48:=============================================> (8 + 0) / 10][Stage 48:=============================================> (8 + 2) / 10] [Stage 49:=============================================> (8 + 0) / 10] [Stage 50:=============================================> (8 + 0) / 10][Stage 50:===================================================> (9 + 1) / 10] [Stage 51:=============================================> (8 + 0) / 10] [Stage 52:=============================================> (8 + 0) / 10] [Stage 53:=============================================> (8 + 0) / 10] [Stage 54:=============================================> (8 + 0) / 10] [Stage 55:=============================================> (8 + 0) / 10][Stage 55:===================================================> (9 + 1) / 10] [Stage 56:=============================================> (8 + 0) / 10] [Stage 57:=============================================> (8 + 0) / 10][Stage 57:=============================================> (8 + 2) / 10] [Stage 58:=============================================> (8 + 0) / 10] [Stage 59:=============================================> (8 + 0) / 10][Stage 59:===================================================> (9 + 1) / 10] [Stage 60:=============================================> (8 + 0) / 10] [Stage 61:=============================================> (8 + 0) / 10][Stage 61:===================================================> (9 + 1) / 10] [Stage 62:=============================================> (8 + 0) / 10] [Stage 63:=============================================> (8 + 0) / 10][Stage 63:===================================================> (9 + 1) / 10] [Stage 64:=============================================> (8 + 0) / 10] [Stage 65:=============================================> (8 + 0) / 10][Stage 65:===================================================> (9 + 1) / 10] [Stage 66:=============================================> (8 + 0) / 10][Stage 66:=============================================> (8 + 2) / 10] [Stage 67:=============================================> (8 + 0) / 10][Stage 67:=============================================> (8 + 2) / 10][Stage 67:===================================================> (9 + 1) / 10] [Stage 68:=============================================> (8 + 0) / 10][Stage 68:===================================================> (9 + 1) / 10] [Stage 69:=============================================> (8 + 0) / 10][Stage 69:=============================================> (8 + 2) / 10][Stage 69:===================================================> (9 + 1) / 10] [Stage 70:=============================================> (8 + 0) / 10] [Stage 71:=============================================> (8 + 0) / 10][Stage 71:=============================================> (8 + 2) / 10][Stage 71:===================================================> (9 + 1) / 10] [Stage 72:=============================================> (8 + 0) / 10] [Stage 73:=============================================> (8 + 0) / 10][Stage 73:===================================================> (9 + 1) / 10] [Stage 74:=============================================> (8 + 0) / 10] [Stage 75:=============================================> (8 + 0) / 10][Stage 75:===================================================> (9 + 1) / 10] [Stage 76:=============================================> (8 + 0) / 10] [Stage 77:=============================================> (8 + 0) / 10][Stage 77:===================================================> (9 + 1) / 10] [Stage 78:=============================================> (8 + 0) / 10] [Stage 79:=============================================> (8 + 0) / 10][Stage 79:=============================================> (8 + 2) / 10] [Stage 80:=============================================> (8 + 0) / 10] [Stage 81:=============================================> (8 + 0) / 10][Stage 81:=============================================> (8 + 2) / 10] [Stage 82:=============================================> (8 + 0) / 10] [Stage 83:=============================================> (8 + 0) / 10][Stage 83:===================================================> (9 + 1) / 10] [Stage 84:=============================================> (8 + 0) / 10] [Stage 85:=============================================> (8 + 0) / 10][Stage 85:===================================================> (9 + 1) / 10] [Stage 86:=============================================> (8 + 0) / 10] [Stage 87:=============================================> (8 + 0) / 10][Stage 87:=============================================> (8 + 2) / 10] [Stage 88:=============================================> (8 + 0) / 10] [Stage 89:=============================================> (8 + 0) / 10][Stage 89:===================================================> (9 + 1) / 10] [Stage 90:=============================================> (8 + 0) / 10] [Stage 91:=============================================> (8 + 0) / 10][Stage 91:=============================================> (8 + 2) / 10] [Stage 92:=============================================> (8 + 0) / 10] [Stage 93:=============================================> (8 + 0) / 10][Stage 93:===================================================> (9 + 1) / 10] [Stage 94:=============================================> (8 + 0) / 10] [Stage 95:=============================================> (8 + 0) / 10][Stage 95:===================================================> (9 + 1) / 10] [Stage 96:=============================================> (8 + 0) / 10] [Stage 97:=============================================> (8 + 0) / 10][Stage 97:=============================================> (8 + 2) / 10] [Stage 98:=============================================> (8 + 0) / 10] [Stage 99:=============================================> (8 + 0) / 10][Stage 99:===================================================> (9 + 1) / 10] [Stage 100:============================================> (8 + 0) / 10] [Stage 101:============================================> (8 + 0) / 10] [Stage 102:============================================> (8 + 0) / 10] [Stage 103:============================================> (8 + 0) / 10][Stage 103:============================================> (8 + 2) / 10] [Stage 104:============================================> (8 + 0) / 10] [Stage 105:============================================> (8 + 0) / 10][Stage 105:==================================================> (9 + 1) / 10] [Stage 106:============================================> (8 + 0) / 10] [Stage 107:============================================> (8 + 0) / 10][Stage 107:============================================> (8 + 2) / 10] [Stage 108:============================================> (8 + 0) / 10] [Stage 109:============================================> (8 + 0) / 10][Stage 109:============================================> (8 + 2) / 10] [Stage 110:============================================> (8 + 0) / 10] [Stage 111:============================================> (8 + 0) / 10][Stage 111:==================================================> (9 + 1) / 10] [Stage 112:============================================> (8 + 0) / 10] [Stage 113:============================================> (8 + 0) / 10][Stage 113:==================================================> (9 + 1) / 10] [Stage 114:============================================> (8 + 0) / 10] [Stage 115:============================================> (8 + 0) / 10][Stage 115:==================================================> (9 + 1) / 10] [Stage 116:============================================> (8 + 0) / 10][Stage 116:============================================> (8 + 2) / 10] [Stage 117:============================================> (8 + 0) / 10][Stage 117:============================================> (8 + 2) / 10][Stage 117:==================================================> (9 + 1) / 10] [Stage 118:============================================> (8 + 0) / 10][Stage 118:============================================> (8 + 2) / 10] [Stage 119:============================================> (8 + 0) / 10][Stage 119:============================================> (8 + 2) / 10] [Stage 120:============================================> (8 + 0) / 10][Stage 120:==================================================> (9 + 1) / 10] [Stage 121:============================================> (8 + 0) / 10][Stage 121:============================================> (8 + 2) / 10] [Stage 122:============================================> (8 + 0) / 10] [Stage 123:============================================> (8 + 0) / 10][Stage 123:============================================> (8 + 2) / 10] [Stage 124:============================================> (8 + 0) / 10] [Stage 125:============================================> (8 + 0) / 10][Stage 125:==================================================> (9 + 1) / 10] [Stage 126:============================================> (8 + 0) / 10] [Stage 127:============================================> (8 + 0) / 10][Stage 127:==================================================> (9 + 1) / 10] [Stage 128:============================================> (8 + 0) / 10] [Stage 129:============================================> (8 + 0) / 10][Stage 129:==================================================> (9 + 1) / 10] [Stage 130:============================================> (8 + 0) / 10] [Stage 131:============================================> (8 + 0) / 10][Stage 131:============================================> (8 + 2) / 10] [Stage 132:============================================> (8 + 0) / 10] [Stage 133:============================================> (8 + 0) / 10][Stage 133:==================================================> (9 + 1) / 10] [Stage 134:============================================> (8 + 0) / 10] [Stage 135:============================================> (8 + 0) / 10][Stage 135:============================================> (8 + 2) / 10] [Stage 136:============================================> (8 + 0) / 10] [Stage 137:============================================> (8 + 0) / 10][Stage 137:============================================> (8 + 2) / 10] [Stage 138:============================================> (8 + 0) / 10] [Stage 139:============================================> (8 + 0) / 10][Stage 139:============================================> (8 + 2) / 10] [Stage 140:============================================> (8 + 0) / 10] [Stage 141:============================================> (8 + 0) / 10][Stage 141:============================================> (8 + 2) / 10] [Stage 142:============================================> (8 + 0) / 10] [Stage 143:============================================> (8 + 0) / 10][Stage 143:============================================> (8 + 2) / 10] [Stage 144:============================================> (8 + 0) / 10] [Stage 145:============================================> (8 + 0) / 10][Stage 145:==================================================> (9 + 1) / 10] [Stage 146:============================================> (8 + 0) / 10] [Stage 147:============================================> (8 + 0) / 10][Stage 147:==================================================> (9 + 1) / 10] [Stage 148:============================================> (8 + 0) / 10] [Stage 149:============================================> (8 + 0) / 10][Stage 149:============================================> (8 + 2) / 10] [Stage 150:============================================> (8 + 0) / 10] [Stage 151:============================================> (8 + 0) / 10][Stage 151:============================================> (8 + 2) / 10] [Stage 152:============================================> (8 + 0) / 10] [Stage 153:============================================> (8 + 0) / 10][Stage 153:==================================================> (9 + 1) / 10] [Stage 154:============================================> (8 + 0) / 10] [Stage 155:============================================> (8 + 0) / 10][Stage 155:============================================> (8 + 2) / 10] [Stage 156:============================================> (8 + 0) / 10] [Stage 157:============================================> (8 + 0) / 10][Stage 157:==================================================> (9 + 1) / 10] [Stage 158:============================================> (8 + 0) / 10] [Stage 159:============================================> (8 + 0) / 10][Stage 159:============================================> (8 + 2) / 10] [Stage 160:============================================> (8 + 0) / 10] [Stage 161:============================================> (8 + 0) / 10] [Stage 162:============================================> (8 + 0) / 10] [Stage 163:============================================> (8 + 0) / 10] [Stage 164:============================================> (8 + 0) / 10] [Stage 165:============================================> (8 + 0) / 10][Stage 165:==================================================> (9 + 1) / 10] [Stage 166:============================================> (8 + 0) / 10][Stage 166:============================================> (8 + 2) / 10] [Stage 167:============================================> (8 + 0) / 10][Stage 167:============================================> (8 + 2) / 10][Stage 167:==================================================> (9 + 1) / 10] [Stage 168:============================================> (8 + 0) / 10][Stage 168:============================================> (8 + 2) / 10] [Stage 169:============================================> (8 + 0) / 10][Stage 169:============================================> (8 + 2) / 10][Stage 169:==================================================> (9 + 1) / 10] [Stage 170:============================================> (8 + 0) / 10][Stage 170:==================================================> (9 + 1) / 10] [Stage 171:============================================> (8 + 0) / 10][Stage 171:==================================================> (9 + 1) / 10] [Stage 172:============================================> (8 + 0) / 10] [Stage 173:============================================> (8 + 0) / 10][Stage 173:============================================> (8 + 2) / 10] [Stage 174:============================================> (8 + 0) / 10] [Stage 175:============================================> (8 + 0) / 10][Stage 175:============================================> (8 + 2) / 10] [Stage 176:============================================> (8 + 0) / 10] [Stage 177:============================================> (8 + 0) / 10][Stage 177:==================================================> (9 + 1) / 10] [Stage 178:============================================> (8 + 0) / 10] [Stage 179:============================================> (8 + 0) / 10][Stage 179:============================================> (8 + 2) / 10] [Stage 180:============================================> (8 + 0) / 10] [Stage 181:============================================> (8 + 0) / 10][Stage 181:==================================================> (9 + 1) / 10] [Stage 182:============================================> (8 + 0) / 10] [Stage 183:============================================> (8 + 0) / 10][Stage 183:============================================> (8 + 2) / 10] [Stage 184:============================================> (8 + 0) / 10] [Stage 185:============================================> (8 + 0) / 10][Stage 185:==================================================> (9 + 1) / 10] [Stage 186:============================================> (8 + 0) / 10] [Stage 187:============================================> (8 + 0) / 10][Stage 187:============================================> (8 + 2) / 10] [Stage 188:============================================> (8 + 0) / 10] [Stage 189:============================================> (8 + 0) / 10][Stage 189:==================================================> (9 + 1) / 10] [Stage 190:============================================> (8 + 0) / 10] [Stage 191:============================================> (8 + 0) / 10][Stage 191:============================================> (8 + 2) / 10] [Stage 192:============================================> (8 + 0) / 10] [Stage 193:============================================> (8 + 0) / 10][Stage 193:============================================> (8 + 2) / 10] [Stage 194:============================================> (8 + 0) / 10] [Stage 195:============================================> (8 + 0) / 10][Stage 195:============================================> (8 + 2) / 10] [Stage 196:============================================> (8 + 0) / 10] [Stage 197:============================================> (8 + 0) / 10] [Stage 198:============================================> (8 + 0) / 10] [Stage 199:============================================> (8 + 0) / 10][Stage 199:==================================================> (9 + 1) / 10] [Stage 200:============================================> (8 + 0) / 10] [Stage 201:============================================> (8 + 0) / 10][Stage 201:==================================================> (9 + 1) / 10] [Stage 202:============================================> (8 + 0) / 10] [Stage 203:============================================> (8 + 0) / 10][Stage 203:==================================================> (9 + 1) / 10] [Stage 204:============================================> (8 + 0) / 10] [Stage 205:============================================> (8 + 0) / 10] [Stage 206:============================================> (8 + 0) / 10] [Stage 207:============================================> (8 + 0) / 10] [Stage 208:============================================> (8 + 0) / 10] [Stage 209:============================================> (8 + 0) / 10][Stage 209:============================================> (8 + 2) / 10] [Stage 210:============================================> (8 + 0) / 10] [Stage 211:============================================> (8 + 0) / 10] [Stage 212:============================================> (8 + 0) / 10] [Stage 213:============================================> (8 + 0) / 10][Stage 213:============================================> (8 + 2) / 10] [Stage 214:============================================> (8 + 0) / 10][Stage 214:============================================> (8 + 2) / 10] [Stage 215:============================================> (8 + 0) / 10][Stage 215:============================================> (8 + 2) / 10][Stage 215:==================================================> (9 + 1) / 10] [Stage 216:============================================> (8 + 0) / 10][Stage 216:============================================> (8 + 2) / 10] [Stage 217:============================================> (8 + 0) / 10][Stage 217:==================================================> (9 + 1) / 10] [Stage 218:============================================> (8 + 0) / 10][Stage 218:============================================> (8 + 2) / 10] [Stage 219:============================================> (8 + 0) / 10][Stage 219:============================================> (8 + 2) / 10] [Stage 220:============================================> (8 + 0) / 10] [Stage 221:============================================> (8 + 0) / 10][Stage 221:==================================================> (9 + 1) / 10] [Stage 222:============================================> (8 + 0) / 10] [Stage 223:============================================> (8 + 0) / 10][Stage 223:============================================> (8 + 2) / 10] [Stage 224:============================================> (8 + 0) / 10] [Stage 225:============================================> (8 + 0) / 10][Stage 225:==================================================> (9 + 1) / 10] [Stage 226:============================================> (8 + 0) / 10] [Stage 227:============================================> (8 + 0) / 10][Stage 227:============================================> (8 + 2) / 10] [Stage 228:============================================> (8 + 0) / 10] [Stage 229:============================================> (8 + 0) / 10][Stage 229:==================================================> (9 + 1) / 10] [Stage 230:============================================> (8 + 0) / 10] [Stage 231:============================================> (8 + 0) / 10][Stage 231:============================================> (8 + 2) / 10] [Stage 232:============================================> (8 + 0) / 10] [Stage 233:============================================> (8 + 0) / 10][Stage 233:==================================================> (9 + 1) / 10] [Stage 234:============================================> (8 + 0) / 10] [Stage 235:============================================> (8 + 0) / 10][Stage 235:============================================> (8 + 2) / 10] [Stage 236:============================================> (8 + 0) / 10] [Stage 237:============================================> (8 + 0) / 10][Stage 237:============================================> (8 + 2) / 10] [Stage 238:============================================> (8 + 0) / 10] [Stage 239:============================================> (8 + 0) / 10][Stage 239:============================================> (8 + 2) / 10] [Stage 240:============================================> (8 + 0) / 10] [Stage 241:============================================> (8 + 0) / 10][Stage 241:============================================> (8 + 2) / 10] [Stage 242:============================================> (8 + 0) / 10] [Stage 243:============================================> (8 + 0) / 10]Exception in thread "streaming-job-executor-0" java.lang.Error: java.lang.InterruptedException
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1148)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:998)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:925)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1.apply(RDD.scala:923)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)
at org.apache.spark.rdd.RDD.foreachPartition(RDD.scala:923)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply$mcV$sp(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.Dataset$$anonfun$foreachPartition$1.apply(Dataset.scala:2305)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2765)
at org.apache.spark.sql.Dataset.foreachPartition(Dataset.scala:2304)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:257)
at SparkCore$$anonfun$main$2.apply(SparkCore.scala:252)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.DStream$$anonfun$foreachRDD$1$$anonfun$apply$mcV$sp$3.apply(DStream.scala:627)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1$$anonfun$apply$mcV$sp$1.apply(ForEachDStream.scala:51)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply$mcV$sp(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at org.apache.spark.streaming.dstream.ForEachDStream$$anonfun$1.apply(ForEachDStream.scala:50)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.Job.run(Job.scala:39)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply$mcV$sp(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler$$anonfun$run$1.apply(JobScheduler.scala:254)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.scheduler.JobScheduler$JobHandler.run(JobScheduler.scala:253)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
... 2 more
start at spark://master:7077
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 21:==================> (12 + 24) / 36][Stage 21:===========================================> (28 + 8) / 36][Stage 22:> (0 + 0) / 36][Stage 22:======================================================> (35 + 1) / 36] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 28:===================================================> (9 + 1) / 10] [Stage 320:> (0 + 0) / 10] [Stage 322:> (0 + 0) / 10] [Stage 326:> (0 + 0) / 10] [Stage 741:> (0 + 0) / 10] [Stage 1401:> (0 + 0) / 10] [Stage 1405:> (0 + 0) / 10] [Stage 1690:============================================> (8 + 2) / 10][Stage 1690:=================================================> (9 + 1) / 10] [Stage 1695:============================================> (8 + 2) / 10][Stage 1695:=================================================> (9 + 1) / 10] [Stage 1697:=================================================> (9 + 1) / 10] [Stage 1699:=================================================> (9 + 1) / 10] [Stage 1720:=================================================> (9 + 1) / 10] [Stage 1726:=================================================> (9 + 1) / 10] [Stage 1733:=================================================> (9 + 1) / 10] [Stage 1743:=================================================> (9 + 1) / 10] [Stage 1762:=================================================> (9 + 1) / 10] [Stage 1788:=================================================> (9 + 1) / 10] [Stage 1813:> (0 + 0) / 10] [Stage 1814:=================================================> (9 + 1) / 10] [Stage 1828:=================================================> (9 + 1) / 10] [Stage 1854:=================================================> (9 + 1) / 10] [Stage 1868:=================================================> (9 + 1) / 10] [Stage 1890:=================================================> (9 + 1) / 10] [Stage 1912:=================================================> (9 + 1) / 10] [Stage 1966:=================================================> (9 + 1) / 10] [Stage 1976:=================================================> (9 + 1) / 10] [Stage 1996:=================================================> (9 + 1) / 10] [Stage 2022:=================================================> (9 + 1) / 10] [Stage 2052:=================================================> (9 + 1) / 10] [Stage 2226:> (0 + 0) / 10] [Stage 2577:============================================> (8 + 2) / 10] [Stage 2590:> (0 + 0) / 10] [Stage 2606:> (0 + 0) / 10] [Stage 2994:> (0 + 0) / 10] [Stage 2996:> (0 + 0) / 10] [Stage 3000:> (0 + 0) / 10] [Stage 3008:> (0 + 0) / 10] [Stage 3016:> (0 + 0) / 10] [Stage 3432:> (0 + 0) / 10] [Stage 3452:> (0 + 0) / 10] [Stage 3489:=================================================> (9 + 1) / 10] [Stage 3490:============================================> (8 + 0) / 10][Stage 3490:============================================> (8 + 2) / 10][Stage 3490:=================================================> (9 + 1) / 10] [Stage 3491:============================================> (8 + 2) / 10][Stage 3491:=================================================> (9 + 1) / 10] [Stage 3497:============================================> (8 + 2) / 10] [Stage 3498:============================================> (8 + 0) / 10][Stage 3498:============================================> (8 + 2) / 10] [Stage 3499:============================================> (8 + 2) / 10] [Stage 3618:> (0 + 0) / 10] [Stage 3624:> (0 + 0) / 10] [Stage 3640:> (0 + 0) / 10] [Stage 3901:=================================================> (9 + 1) / 10] [Stage 3982:> (0 + 0) / 10] [Stage 4150:===========================> (5 + 5) / 10] [Stage 4373:> (0 + 0) / 10] [Stage 4375:> (0 + 0) / 10] [Stage 4782:============================================> (8 + 2) / 10] [Stage 4797:> (0 + 0) / 10] [Stage 4801:> (0 + 0) / 10] [Stage 4810:============================================> (8 + 2) / 10][Stage 4810:=================================================> (9 + 1) / 10] [Stage 4811:============================================> (8 + 0) / 10][Stage 4811:============================================> (8 + 2) / 10][Stage 4811:=================================================> (9 + 1) / 10] [Stage 4812:============================================> (8 + 2) / 10] [Stage 5261:> (0 + 0) / 10] [Stage 5265:> (0 + 0) / 10] [Stage 5275:> (0 + 0) / 10] [Stage 5279:> (0 + 0) / 10] [Stage 5297:> (0 + 0) / 10] [Stage 5658:======================================> (7 + 3) / 10] [Stage 5693:=================================================> (9 + 1) / 10] [Stage 5705:=================================================> (9 + 1) / 10] [Stage 5708:=================================================> (9 + 1) / 10] [Stage 5775:> (0 + 0) / 10] [Stage 5786:=================================================> (9 + 1) / 10] [Stage 5816:=================================================> (9 + 1) / 10] [Stage 5886:=================================================> (9 + 1) / 10] [Stage 5891:=================================================> (9 + 1) / 10] [Stage 5965:=================================================> (9 + 1) / 10] [Stage 5991:=================================================> (9 + 1) / 10] [Stage 6004:=================================================> (9 + 1) / 10] [Stage 6024:=================================================> (9 + 1) / 10] [Stage 6205:> (0 + 0) / 10] [Stage 6213:> (0 + 0) / 10] [Stage 6223:> (0 + 0) / 10] [Stage 6481:============================================> (8 + 2) / 10] [Stage 6656:> (0 + 0) / 10] [Stage 6658:> (0 + 0) / 10] [Stage 6660:> (0 + 0) / 10] [Stage 6662:> (0 + 0) / 10] [Stage 7073:============================================> (8 + 2) / 10][Stage 7073:=================================================> (9 + 1) / 10] [Stage 7086:> (0 + 0) / 10] [Stage 7088:> (0 + 0) / 10] [Stage 7100:> (0 + 0) / 10] [Stage 7102:> (0 + 0) / 10] [Stage 7121:============================================> (8 + 2) / 10][Stage 7121:=================================================> (9 + 1) / 10] [Stage 7122:============================================> (8 + 0) / 10][Stage 7122:============================================> (8 + 2) / 10] [Stage 7123:============================================> (8 + 2) / 10] [Stage 7430:=================================================> (9 + 1) / 10] [Stage 7466:> (0 + 0) / 10] [Stage 7467:> (0 + 0) / 10] [Stage 7477:> (0 + 0) / 10] [Stage 7806:> (0 + 0) / 10] [Stage 7814:> (0 + 0) / 10] [Stage 7816:> (0 + 0) / 10] [Stage 7994:============================================> (8 + 2) / 10][Stage 7994:=================================================> (9 + 1) / 10] [Stage 7995:============================================> (8 + 2) / 10] [Stage 8236:> (0 + 0) / 10] [Stage 8389:============================================> (8 + 2) / 10] [Stage 8559:============================================> (8 + 2) / 10] [Stage 8587:> (0 + 0) / 10] [Stage 8592:> (0 + 0) / 10] [Stage 8612:============================================> (8 + 2) / 10] [Stage 8648:============================================> (8 + 2) / 10] [Stage 8850:> (0 + 0) / 10] [Stage 8858:> (0 + 0) / 10] [Stage 8860:> (0 + 0) / 10] [Stage 8868:> (0 + 0) / 10] [Stage 8870:> (0 + 0) / 10] [Stage 9016:=================================================> (9 + 1) / 10] [Stage 9024:============================================> (8 + 2) / 10][Stage 9024:=================================================> (9 + 1) / 10] [Stage 9025:============================================> (8 + 2) / 10][Stage 9025:=================================================> (9 + 1) / 10] [Stage 9060:=================================================> (9 + 1) / 10] [Stage 9070:=================================================> (9 + 1) / 10] [Stage 9074:=================================================> (9 + 1) / 10] [Stage 9082:=================================================> (9 + 1) / 10] [Stage 9153:=================================================> (9 + 1) / 10] [Stage 9183:=================================================> (9 + 1) / 10] [Stage 9221:=================================================> (9 + 1) / 10] [Stage 9255:=================================================> (9 + 1) / 10] [Stage 9279:=================================================> (9 + 1) / 10] [Stage 9290:> (0 + 0) / 10] [Stage 9294:> (0 + 0) / 10] [Stage 9298:> (0 + 0) / 10] [Stage 9344:===========================> (5 + 5) / 10][Stage 9344:=================================> (6 + 4) / 10][Stage 9344:=================================================> (9 + 1) / 10] [Stage 9666:> (0 + 0) / 10] [Stage 9668:> (0 + 0) / 10] [Stage 9692:============================================> (8 + 2) / 10] [Stage 10085:> (0 + 0) / 10] [Stage 10182:===========================================> (8 + 2) / 10][Stage 10182:================================================> (9 + 1) / 10] [Stage 10188:=====================================> (7 + 3) / 10][Stage 10188:================================================> (9 + 1) / 10] [Stage 10243:================================================> (9 + 1) / 10] [Stage 10302:================================================> (9 + 1) / 10] [Stage 10339:================================================> (9 + 1) / 10] [Stage 10343:================================> (6 + 4) / 10][Stage 10343:=====================================> (7 + 3) / 10][Stage 10343:================================================> (9 + 1) / 10] [Stage 10345:===========================================> (8 + 2) / 10] [Stage 10347:================================================> (9 + 1) / 10] [Stage 10353:===========================> (5 + 5) / 10][Stage 10353:================================> (6 + 4) / 10][Stage 10353:=====================================> (7 + 3) / 10][Stage 10353:===========================================> (8 + 2) / 10] [Stage 10355:================================================> (9 + 1) / 10] [Stage 10357:================================================> (9 + 1) / 10] [Stage 10359:=====================================> (7 + 3) / 10][Stage 10359:===========================================> (8 + 2) / 10] [Stage 10361:===========================================> (8 + 2) / 10] [Stage 10371:================================================> (9 + 1) / 10] [Stage 10377:================================================> (9 + 1) / 10] [Stage 10387:================================================> (9 + 1) / 10] [Stage 10395:================================================> (9 + 1) / 10] [Stage 10409:================================================> (9 + 1) / 10] [Stage 10430:================================================> (9 + 1) / 10] [Stage 10497:================================================> (9 + 1) / 10] [Stage 10525:================================================> (9 + 1) / 10] [Stage 10590:================================================> (9 + 1) / 10] [Stage 10597:================================================> (9 + 1) / 10] [Stage 10614:================================================> (9 + 1) / 10] [Stage 10635:================================================> (9 + 1) / 10] [Stage 10675:================================================> (9 + 1) / 10] [Stage 10681:================================================> (9 + 1) / 10] [Stage 10689:=====================================> (7 + 3) / 10][Stage 10689:===========================================> (8 + 2) / 10] [Stage 10690:================================> (6 + 0) / 10][Stage 10690:================================> (6 + 4) / 10][Stage 10690:=====================================> (7 + 3) / 10][Stage 10690:================================================> (9 + 1) / 10] [Stage 10691:================================> (6 + 4) / 10][Stage 10691:=====================================> (7 + 3) / 10][Stage 10691:================================================> (9 + 1) / 10] [Stage 10693:================================================> (9 + 1) / 10] [Stage 10715:================================> (6 + 4) / 10][Stage 10715:=====================================> (7 + 3) / 10] [Stage 10731:================================================> (9 + 1) / 10] [Stage 10748:================================================> (9 + 1) / 10] [Stage 10783:================================================> (9 + 1) / 10] [Stage 11384:> (0 + 0) / 10] [Stage 11388:> (0 + 0) / 10] [Stage 11390:> (0 + 0) / 10] [Stage 11431:===========================================> (8 + 2) / 10] [Stage 11740:> (0 + 0) / 10] [Stage 11750:> (0 + 0) / 10] [Stage 11762:> (0 + 0) / 10] [Stage 11769:================================================> (9 + 1) / 10] Exception in thread "main" org.apache.spark.SparkException: ArrayBuffer(java.nio.channels.ClosedChannelException, java.nio.channels.ClosedChannelException, java.nio.channels.ClosedChannelException, java.nio.channels.ClosedChannelException)
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.latestLeaderOffsets(DirectKafkaInputDStream.scala:133)
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.compute(DirectKafkaInputDStream.scala:158)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.MappedDStream.compute(MappedDStream.scala:36)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.ForEachDStream.generateJob(ForEachDStream.scala:48)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:117)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:116)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241)
at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104)
at org.apache.spark.streaming.DStreamGraph.generateJobs(DStreamGraph.scala:116)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:249)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:247)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.JobGenerator.generateJobs(JobGenerator.scala:247)
at org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:183)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:89)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:88)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
start at spark://master:7077
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
[Stage 21:==================> (12 + 24) / 36] Accuracy = 0.7523809523809524
Precision(0.0) = 0.5714285714285714
Precision(1.0) = 0.7619047619047619
Precision(2.0) = 0.9523809523809523
Precision(3.0) = 0.8571428571428571
Precision(4.0) = 0.6190476190476191
Recall(0.0) = 0.8571428571428571
Recall(1.0) = 0.8888888888888888
Recall(2.0) = 0.5882352941176471
Recall(3.0) = 0.72
Recall(4.0) = 0.9285714285714286
Start Reading Stream Text
[Stage 28:=====> (1 + 9) / 10][Stage 28:=============================================> (8 + 2) / 10] [Stage 32:===================================================> (9 + 1) / 10] [Stage 36:===================================================> (9 + 1) / 10] [Stage 89:===================================================> (9 + 1) / 10] [Stage 103:==================================================> (9 + 1) / 10] [Stage 139:==================================================> (9 + 1) / 10] [Stage 178:==================================================> (9 + 1) / 10] [Stage 184:==================================================> (9 + 1) / 10] [Stage 188:==================================================> (9 + 1) / 10] [Stage 198:==================================================> (9 + 1) / 10] [Stage 202:==================================================> (9 + 1) / 10] [Stage 204:==================================================> (9 + 1) / 10] [Stage 246:==================================================> (9 + 1) / 10] [Stage 262:==================================================> (9 + 1) / 10] [Stage 271:> (0 + 0) / 10] [Stage 286:==================================================> (9 + 1) / 10] [Stage 316:==================================================> (9 + 1) / 10] [Stage 366:==================================================> (9 + 1) / 10] [Stage 368:==================================================> (9 + 1) / 10] [Stage 390:==================================================> (9 + 1) / 10] [Stage 400:==================================================> (9 + 1) / 10] [Stage 430:==================================================> (9 + 1) / 10] [Stage 512:==================================================> (9 + 1) / 10] [Stage 520:==================================================> (9 + 1) / 10] [Stage 529:==================================================> (9 + 1) / 10] [Stage 549:==================================================> (9 + 1) / 10] [Stage 557:==================================================> (9 + 1) / 10] [Stage 575:==================================================> (9 + 1) / 10] [Stage 591:==================================================> (9 + 1) / 10] [Stage 595:==================================================> (9 + 1) / 10] [Stage 615:==================================================> (9 + 1) / 10] [Stage 641:==================================================> (9 + 1) / 10] [Stage 663:==================================================> (9 + 1) / 10] [Stage 677:==================================================> (9 + 1) / 10] [Stage 701:==================================================> (9 + 1) / 10] [Stage 705:==================================================> (9 + 1) / 10] [Stage 716:> (0 + 0) / 10] [Stage 720:> (0 + 0) / 10] [Stage 727:==================================================> (9 + 1) / 10] [Stage 731:==================================================> (9 + 1) / 10] [Stage 743:==================================================> (9 + 1) / 10] [Stage 747:==================================================> (9 + 1) / 10] [Stage 757:==================================================> (9 + 1) / 10] [Stage 773:==================================================> (9 + 1) / 10] [Stage 791:==================================================> (9 + 1) / 10] [Stage 797:==================================================> (9 + 1) / 10] [Stage 839:==================================================> (9 + 1) / 10] [Stage 879:==================================================> (9 + 1) / 10] [Stage 965:==================================================> (9 + 1) / 10] [Stage 979:==================================================> (9 + 1) / 10] [Stage 987:============================================> (8 + 2) / 10] [Stage 995:==================================================> (9 + 1) / 10] [Stage 1031:=================================================> (9 + 1) / 10] [Stage 1094:> (0 + 0) / 10] [Stage 1480:> (0 + 0) / 10] [Stage 1498:> (0 + 0) / 10] [Stage 1500:> (0 + 0) / 10] [Stage 1504:> (0 + 0) / 10] [Stage 1508:> (0 + 0) / 10] [Stage 1636:=================================================> (9 + 1) / 10] [Stage 2164:> (0 + 0) / 10] [Stage 2168:> (0 + 0) / 10] [Stage 2190:> (0 + 0) / 10] [Stage 2570:> (0 + 0) / 10] [Stage 2617:======================================> (7 + 3) / 10] [Stage 2920:> (0 + 0) / 10] [Stage 2933:=================================================> (9 + 1) / 10] [Stage 3021:=================================================> (9 + 1) / 10] [Stage 3224:> (0 + 0) / 10] [Stage 3350:> (0 + 0) / 10] [Stage 3354:> (0 + 0) / 10] [Stage 3364:> (0 + 0) / 10] [Stage 3370:> (0 + 0) / 10] [Stage 3635:> (0 + 0) / 10] [Stage 3940:=================================================> (9 + 1) / 10] [Stage 3962:=================================================> (9 + 1) / 10] [Stage 3986:=================================================> (9 + 1) / 10] [Stage 4062:=================================================> (9 + 1) / 10] [Stage 4098:=================================================> (9 + 1) / 10] [Stage 4103:> (0 + 0) / 10] [Stage 4111:> (0 + 0) / 10] [Stage 4200:=================================================> (9 + 1) / 10] [Stage 4254:============================================> (8 + 2) / 10][Stage 4254:=================================================> (9 + 1) / 10] [Stage 4256:============================================> (8 + 2) / 10][Stage 4256:=================================================> (9 + 1) / 10] [Stage 4260:======================================> (7 + 3) / 10][Stage 4260:=================================================> (9 + 1) / 10] [Stage 4326:=================================================> (9 + 1) / 10] [Stage 4595:> (0 + 0) / 10] [Stage 4597:> (0 + 0) / 10] [Stage 4601:> (0 + 0) / 10] [Stage 4615:> (0 + 0) / 10] [Stage 4870:=================================================> (9 + 1) / 10] [Stage 4922:============================================> (8 + 2) / 10] [Stage 5109:> (0 + 0) / 10] [Stage 5119:> (0 + 0) / 10] [Stage 5123:> (0 + 0) / 10] [Stage 5213:> (0 + 0) / 10] [Stage 5215:> (0 + 0) / 10] [Stage 5225:> (0 + 0) / 10] [Stage 5231:> (0 + 0) / 10] [Stage 5233:> (0 + 0) / 10] [Stage 5300:============================================> (8 + 2) / 10] [Stage 5306:============================================> (8 + 2) / 10] [Stage 5610:> (0 + 0) / 10] [Stage 5612:> (0 + 0) / 10] [Stage 5614:> (0 + 0) / 10] [Stage 5616:> (0 + 0) / 10] [Stage 5617:> (0 + 0) / 10] [Stage 5820:============================================> (8 + 2) / 10] [Stage 5987:> (0 + 0) / 10] [Stage 5989:> (0 + 0) / 10] [Stage 6033:> (0 + 0) / 10] [Stage 6089:============================================> (8 + 2) / 10] [Stage 6108:============================================> (8 + 2) / 10][Stage 6108:=================================================> (9 + 1) / 10] [Stage 6286:============================================> (8 + 2) / 10] [Stage 6355:> (0 + 0) / 10] [Stage 6357:> (0 + 0) / 10] [Stage 6361:> (0 + 0) / 10] [Stage 6371:> (0 + 0) / 10] [Stage 6379:> (0 + 0) / 10] [Stage 6600:=================================================> (9 + 1) / 10] [Stage 6739:============================================> (8 + 2) / 10] [Stage 6758:============================================> (8 + 2) / 10][Stage 6758:=================================================> (9 + 1) / 10] [Stage 6765:> (0 + 0) / 10] [Stage 6769:> (0 + 0) / 10] [Stage 6779:> (0 + 0) / 10] [Stage 6783:> (0 + 0) / 10] [Stage 6785:> (0 + 0) / 10] [Stage 6975:> (0 + 0) / 10] [Stage 6987:> (0 + 0) / 10] [Stage 6998:============================================> (8 + 2) / 10] [Stage 7342:============================================> (8 + 2) / 10] [Stage 7388:============================================> (8 + 2) / 10] [Stage 7410:=================================================> (9 + 1) / 10] [Stage 7421:> (0 + 0) / 10] [Stage 7425:> (0 + 0) / 10] [Stage 7435:> (0 + 0) / 10] [Stage 7598:============================================> (8 + 2) / 10] [Stage 7629:============================================> (8 + 2) / 10] [Stage 7825:============================================> (8 + 2) / 10] [Stage 7891:> (0 + 0) / 10] [Stage 7895:> (0 + 0) / 10] [Stage 7903:> (0 + 0) / 10] [Stage 7982:============================================> (8 + 2) / 10] [Stage 8090:============================================> (8 + 2) / 10][Stage 8090:=================================================> (9 + 1) / 10] [Stage 8132:=================================================> (9 + 1) / 10] [Stage 8218:============================================> (8 + 2) / 10][Stage 8218:=================================================> (9 + 1) / 10] [Stage 8262:=================================================> (9 + 1) / 10] [Stage 8304:============================================> (8 + 2) / 10][Stage 8304:=================================================> (9 + 1) / 10] [Stage 8323:> (0 + 0) / 10] [Stage 8344:============================================> (8 + 2) / 10] [Stage 8366:=================================================> (9 + 1) / 10] [Stage 8374:=================================================> (9 + 1) / 10] [Stage 8402:=================================================> (9 + 1) / 10] [Stage 8421:============================================> (8 + 2) / 10] [Stage 8466:============================================> (8 + 2) / 10] [Stage 8592:=================================================> (9 + 1) / 10] [Stage 8763:> (0 + 0) / 10] [Stage 8769:> (0 + 0) / 10] [Stage 8842:============================================> (8 + 2) / 10] [Stage 8850:============================================> (8 + 2) / 10] [Stage 8912:============================================> (8 + 2) / 10] [Stage 8946:============================================> (8 + 2) / 10][Stage 8946:=================================================> (9 + 1) / 10] [Stage 9194:============================================> (8 + 2) / 10] [Stage 9205:> (0 + 0) / 10] [Stage 9213:> (0 + 0) / 10] [Stage 9278:============================================> (8 + 2) / 10] [Stage 9280:============================================> (8 + 2) / 10][Stage 9280:=================================================> (9 + 1) / 10] [Stage 9328:============================================> (8 + 2) / 10] [Stage 9410:=================================================> (9 + 1) / 10] [Stage 9580:============================================> (8 + 2) / 10][Stage 9580:=================================================> (9 + 1) / 10] [Stage 9599:============================================> (8 + 2) / 10] [Stage 9652:============================================> (8 + 2) / 10] [Stage 9685:> (0 + 0) / 10] [Stage 9707:> (0 + 0) / 10] [Stage 9804:============================================> (8 + 2) / 10] [Stage 9896:============================================> (8 + 2) / 10] [Stage 9944:============================================> (8 + 2) / 10] [Stage 10058:================================================> (9 + 1) / 10] Exception in thread "main" org.apache.spark.SparkException: ArrayBuffer(java.nio.channels.ClosedChannelException, org.apache.spark.SparkException: Couldn't find leader offsets for Set([tweet,1], [tweet,6]))
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.latestLeaderOffsets(DirectKafkaInputDStream.scala:133)
at org.apache.spark.streaming.kafka.DirectKafkaInputDStream.compute(DirectKafkaInputDStream.scala:158)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.MappedDStream.compute(MappedDStream.scala:36)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1$$anonfun$apply$7.apply(DStream.scala:341)
at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1$$anonfun$1.apply(DStream.scala:340)
at org.apache.spark.streaming.dstream.DStream.createRDDWithLocalProperties(DStream.scala:415)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:335)
at org.apache.spark.streaming.dstream.DStream$$anonfun$getOrCompute$1.apply(DStream.scala:333)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.streaming.dstream.DStream.getOrCompute(DStream.scala:330)
at org.apache.spark.streaming.dstream.ForEachDStream.generateJob(ForEachDStream.scala:48)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:117)
at org.apache.spark.streaming.DStreamGraph$$anonfun$1.apply(DStreamGraph.scala:116)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241)
at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104)
at org.apache.spark.streaming.DStreamGraph.generateJobs(DStreamGraph.scala:116)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:249)
at org.apache.spark.streaming.scheduler.JobGenerator$$anonfun$3.apply(JobGenerator.scala:247)
at scala.util.Try$.apply(Try.scala:192)
at org.apache.spark.streaming.scheduler.JobGenerator.generateJobs(JobGenerator.scala:247)
at org.apache.spark.streaming.scheduler.JobGenerator.org$apache$spark$streaming$scheduler$JobGenerator$$processEvent(JobGenerator.scala:183)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:89)
at org.apache.spark.streaming.scheduler.JobGenerator$$anon$1.onReceive(JobGenerator.scala:88)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)