forked from SERG-Delft/phd-thesis-template
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dissertation.bib
3291 lines (3009 loc) · 145 KB
/
dissertation.bib
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
@inproceedings{YZW17descry,
title = {DESCRY: reproducing system-level concurrency failures},
author = {Yu, Tingting and Zaman, Tarannum S and Wang, Chao},
booktitle = {Proceedings of the 2017 11th Joint Meeting on Foundations of Software Engineering},
pages = {694--704},
year = {2017},
organization = {ACM}
}
@article{Shamshiri2018,
abstract = {Achieving high structural coverage is an important aim in software testing. Several search-based techniques have proved successful at automatically generating tests that achieve high coverage. However, despite the well- established arguments behind using evolutionary search algorithms (e.g., genetic algorithms) in preference to random search, it remains an open question whether the benefits can actually be observed in practice when generating unit test suites for object-oriented classes. In this paper, we report an empirical study on the effects of using a genetic algorithm (GA) to generate test suites over generating test suites incrementally with random search, by applying the EvoSuite unit test suite generator to 1,000 classes randomly selected from the SF110 corpus of open source projects. Surprisingly, the results show little difference between the coverage achieved by test suites generated with evolutionary search compared to those generated using random search. A detailed analysis reveals that the genetic algorithm covers more branches of the type where standard fitness functions provide guidance. In practice, however, we observed that the vast majority of branches in the analyzed projects provide no such guidance.},
author = {Shamshiri, Sina and Rojas, Jos{\'{e}} Miguel and Gazzola, Luca and Fraser, Gordon and McMinn, Phil and Mariani, Leonardo and Arcuri, Andrea},
doi = {10.1002/stvr.1660},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Shamshiri et al. - 2018 - Random or evolutionary search for object-oriented test suite generation.pdf:pdf},
isbn = {9781450334723},
issn = {09600833},
journal = {Software Testing, Verification and Reliability},
keywords = {Automated software testing,Automated test generation,Chemical reaction optimization,Genetic algorithms,Random search,Search-based software testing},
month = {jun},
number = {4},
pages = {e1660},
title = {{Random or evolutionary search for object-oriented test suite generation?}},
volume = {28},
year = {2018}
}
@inproceedings{Hashim2005,
abstract = {This paper presents an improvement to existing class test ordering stategies by including coupling measures to reduce non-determinism and decrease the number of stubs to be produced. Our novel strategy aims to lift the known methods from class hierarchies to component-based product lines in which so-called connectors between components are key entities for structuring, assembling and integrating software architectures, and in which tighter coupling of classes inside components and looser coupling between classes in different components are fundamental to component-based architecture composition. The paper also evaluates the new method and confirms the expected reductions in the number of stubs generated and the improved test ordering. Two recent test order strategies for C++ are compared with our novel approach through experiments conducted on two mini product lines written in Java. The new proposed test ordering strategy is more deterministic through measuring coupling, which is at the heart of component-based product line architectures. {\&}copy; 2005 IEEE.},
author = {Hashim, N.L. and Schmidt, H.W. and Ramakrishnan, Sita},
booktitle = {Fifth International Conference on Quality Software (QSIC'05)},
doi = {10.1109/QSIC.2005.64},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Hashim, Schmidt, Ramakrishnan - 2005 - Test Order for Class-based Integration Testing of Java Applications.pdf:pdf},
isbn = {0-7695-2472-9},
issn = {15506002},
mendeley-groups = {Software Testing/Integration Testing},
pages = {11--18},
publisher = {IEEE},
title = {{Test Order for Class-based Integration Testing of Java Applications}},
volume = {2005},
year = {2005}
}
@inproceedings{Artzi2008,
author = {Artzi, Shay and Kim, Sunghun and Ernst, Michael D.},
title = {ReCrash: Making Software Failures Reproducible by Preserving Object States},
booktitle = {Proceedings of the 22Nd European Conference on Object-Oriented Programming},
series = {ECOOP '08},
year = {2008},
isbn = {978-3-540-70591-8},
pages = {542--565},
doi = {10.1007/978-3-540-70592-5_23},
publisher = {Springer-Verlag},
address = {Berlin, Heidelberg},
keywords = {Fault, bug, capture, crash, failure, object, replay, reproducing, test generation}
}
@inproceedings{Rossler2013,
author = {R{\"{o}}{\ss}ler, Jeremias and Zeller, Andreas and Fraser, Gordon and Zamfir, Cristian and Candea, George},
booktitle = {Proc. International Conference on Software Testing, Verification and Validation (ICST)},
doi = {10.1109/ICST.2013.18},
isbn = {978-0-7695-4968-2},
issn = {2159-4848},
pages = {114--123},
publisher = {IEEE},
title = {{Reconstructing core dumps}},
year = {2013}
}
@article{Fraser2015a,
author = {Fraser, Gordon and Arcuri, Andrea},
doi = {10.1007/s10664-013-9299-z},
journal = {EMSE},
month = {jun},
number = {3},
pages = {783--812},
publisher = {Springer},
title = {{Achieving scalable mutation-based generation of whole test suites}},
volume = {20},
year = {2015}
}
@inproceedings{DBLP:conf/iwpc/WhiteVJBP15,
author = {Martin White and
Mario Linares V{\'{a}}squez and
Peter Johnson and
Carlos Bernal{-}C{\'{a}}rdenas and
Denys Poshyvanyk},
title = {Generating reproducible and replayable bug reports from Android application
crashes},
booktitle = {Proceedings of the 2015 {IEEE} 23rd International Conference on Program
Comprehension, {ICPC} 2015, Florence/Firenze, Italy, May 16-24, 2015},
pages = {48--59},
year = {2015},
crossref = {DBLP:conf/iwpc/2015},
doi = {10.1109/ICPC.2015.14}
}
@misc{mavenDep,
author = {{Apache Maven Project}},
title = {{Introduction to the Dependency Mechanism}},
howpublished = {\url{https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html}},
year = {2018},
note = {[Online; accessed 25-January-2018]}
}
@inproceedings{Sim2003,
author = {Sim, Susan Elliott and Easterbrook, Steve and Holt, Richard C},
booktitle = {Proceedings of the 25th International Conference on Software Engineering},
isbn = {0-7695-1877-X},
pages = {74--83},
publisher = {IEEE Computer Society},
series = {ICSE '03},
title = {{Using Benchmarking to Advance Research: A Challenge to Software Engineering}},
year = {2003},
address = {Portland, Oregon, USA}
}
@inproceedings{Alexander2000,
author = {Alexander, R.T. and Offutt, A.J.},
booktitle = {Proceedings 11th International Symposium on Software Reliability Engineering. ISSRE 2000},
doi = {10.1109/ISSRE.2000.885857},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Alexander, Offutt - 2000 - Criteria for testing polymorphic relationships.pdf:pdf},
isbn = {0-7695-0807-3},
mendeley-groups = {Software Testing/Integration Testing},
pages = {15--23},
publisher = {IEEE Comput. Soc},
title = {{Criteria for testing polymorphic relationships}},
year = {2000}
}
@book{Baier2007,
Author = {Baier, Christel and Katoen, Joost-Pieter},
Isbn = {978-0-262-02649-9},
Owner = {Maxime},
Publisher = {MIT Press},
Shelf = {F2},
Timestamp = {2009.04.08},
Title = {Principles of Model Checking},
Year = {2007}
}
@inproceedings{MarkLoopFlag,
title = {Evolutionary testing in the presence of loop-assigned flags: A testability transformation approach},
author = {Baresel, Andr{\'e} and Binkley, David and Harman, Mark and Korel, Bogdan},
booktitle = {ACM SIGSOFT Software Engineering Notes},
volume = {29},
pages = {108--118},
year = {2004},
publisher = {{ACM}},
organization = {ACM},
address = {Boston, Massachusetts, USA}
}
@inproceedings{Goldberg:1987,
author = {Goldberg, David E. and Richardson, Jon},
title = {Genetic Algorithms with Sharing for Multimodal Function Optimization},
booktitle = {Proc. Int'l Conf. on Genetic Algorithms and Their Application},
year = {1987},
isbn = {0-8058-0158-8},
location = {Cambridge, Massachusetts, USA},
pages = {41--49},
numpages = {9},
-url = {http://dl.acm.org/citation.cfm?id=42512.42519},
acmid = {42519},
publisher = {L. Erlbaum Associates Inc.},
-address = {Hillsdale, NJ, USA}
}
@inproceedings{Kracht2014,
author = {Kracht, Jeshua S. and Petrovic, Jacob Z. and Walcott-Justice, Kristen R.},
booktitle = {QSIC'14},
doi = {10.1109/QSIC.2014.33},
month = {oct},
pages = {256--265},
publisher = {IEEE},
title = {{Empirically Evaluating the Quality of Automatically Generated and Manually Written Test Suites}},
year = {2014}
}
@article{vcrepinvsek2013,
author = {\v{C}repin\v{s}ek, Matej and Liu, Shih-Hsi and Mernik, Marjan},
title = {Exploration and exploitation in evolutionary algorithms: A survey},
year = {2013},
issue_date = {June 2013},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {45},
number = {3},
issn = {0360-0300},
url = {https://doi.org/10.1145/2480741.2480752},
journal = {ACM Comput. Surv.},
numpages = {33}
}
@inproceedings{martinez2016astor,
title = {Astor: A program repair library for java},
author = {Martinez, Matias and Monperrus, Martin},
booktitle = {Proceedings of the 25th International Symposium on Software Testing and Analysis},
pages = {441--444},
year = {2016},
organization = {ACM}
}
@inproceedings{Weeratunge2010,
author = {Weeratunge, Dasarath and Zhang, Xiangyu and Jagannathan, Suresh},
title = {Analyzing Multicore Dumps to Facilitate Concurrency Bug Reproduction},
booktitle = {Proceedings of the Fifteenth Edition of ASPLOS on Architectural Support for Programming Languages and Operating Systems},
series = {ASPLOS XV},
year = {2010},
isbn = {978-1-60558-839-1},
pages = {155--166},
doi = {10.1145/1736020.1736039},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {concurrency bugs, execution indexing, multi-core, reproduction}
}
@inproceedings{DaVeigaCabral2010,
abstract = {In the context of Object-Oriented software, many works have investigated the Class Integration and Test Order (CITO) problem, proposing solutions to determine test orders for the integration test of the program classes. The existing approaches based on graphs can generate solutions that are sub-optimal, and do not consider the different factors and measures that can affect the stubbing process. To overcome this limitation, solutions based on Genetic Algorithms (GA) have presented promising results. However, the determination of a cost function, which is able to generate the best solutions, is not always a trivial task, mainly for complex systems with a great number of measures. Therefore, we introduce, in this paper, a multi-objective optimization approach to better represent the CITO problem. The approach generates a set of good solutions that achieve a balanced compromise between the different measures (objectives). It was implemented by a Pareto Ant Colony (P-ACO) algorithm, which is described in detail. The algorithm was used in a set of real programs and the obtained results are compared to the GA results. The results allow discussing the difference between single and multi-objective approaches especially for complex systems with a greater number of dependencies among the classes. {\textcopyright} 2010 IFIP International Federation for Information Processing.},
author = {{da Veiga Cabral}, Rafael and Pozo, Aurora and Vergilio, Silvia Regina},
booktitle = {IFIP International Conference on Testing Software and Systems},
doi = {10.1007/978-3-642-16573-3_3},
file = {:Users/xavierdevroey/Dropbox/These/Articles/da Veiga Cabral, Pozo, Vergilio - 2010 - A Pareto Ant Colony Algorithm Applied to the Class Integration and Test Order Problem.pdf:pdf},
isbn = {3642165729},
issn = {03029743},
keywords = {Integration testing,ant colony algorithm,multi-objective,object-oriented software},
mendeley-groups = {Software Testing/Integration Testing},
pages = {16--29},
publisher = {Springer},
series = {ICTSS 2010},
title = {{A Pareto Ant Colony Algorithm Applied to the Class Integration and Test Order Problem}},
volume = {6435 LNCS},
year = {2010}
}
@article{delamaro2001interface,
title = {Interface mutation: An approach for integration testing},
author = {Delamaro, Marcio Eduardo and Maidonado, JC and Mathur, Aditya P.},
journal = {IEEE transactions on software engineering},
volume = {27},
number = {3},
pages = {228--247},
year = {2001},
publisher = {IEEE}
}
@inproceedings{Steven2000,
author = {Steven, John and Chandra, Pravir and Fleck, Bob and Podgurski, Andy},
title = {jRapture: A Capture/Replay Tool for Observation-based Testing},
booktitle = {Proceedings of the 2000 ACM SIGSOFT International Symposium on Software Testing and Analysis},
series = {ISSTA '00},
year = {2000},
isbn = {1-58113-266-2},
pages = {158--167},
doi = {10.1145/347324.348993},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {Java, capture/replay, execution profiling, observation-based testing, software testing}
}
@inproceedings{Kifetew2014,
author = {Kifetew, Fitsum Meshesha and Jin, Wei and Tiella, Roberto and Orso, Alessandro and Tonella, Paolo},
title = {Reproducing Field Failures for Programs with Complex Grammar-Based Input},
booktitle = {Proceedings of the 2014 IEEE International Conference on Software Testing, Verification, and Validation},
series = {ICST '14},
year = {2014},
isbn = {978-1-4799-2255-0},
pages = {163--172},
doi = {10.1109/ICST.2014.29},
publisher = {IEEE Computer Society},
address = {Washington, DC, USA},
keywords = {Field failures, failure reproduction, genetic programming, search-based software engineering}
}
@dataset{zenodoRP,
author = {Derakhshanfar, Pouria and
Devroey, Xavier and
Zaidman, Andy and
van Deursen, Arie and
Panichella, Annibale},
title = {{Replication package of "Good Things Come In
Threes: Improving Search-based Crash Reproduction
With Helper Objectives"}},
month = aug,
year = 2020,
publisher = {Zenodo},
version = {1.0},
doi = {10.5281/zenodo.3979097},
url = {https://doi.org/10.5281/zenodo.3979097}
}
@article{Derakhshanfar2019a,
archivePrefix = {arXiv},
arxivId = {cs.SE/1912.04606},
author = {Derakhshanfar, Pouria and Devroey, Xavier and Perrouin, Gilles and Zaidman, Andy and van Deursen, Arie},
eprint = {1912.04606},
journal = {arXiv:1912.04606 [cs.SE]},
primaryClass = {cs.SE},
title = {{Search-based Crash Reproduction using Behavioral Model Seeding}},
year = {2019},
note = {(submitted to STVR)}
}
@article{deb2002fast,
title = {A fast and elitist multiobjective genetic algorithm: NSGA-II},
author = {Deb, Kalyanmoy and Pratap, Amrit and Agarwal, Sameer and Meyarivan, TAMT},
journal = {IEEE transactions on evolutionary computation},
volume = {6},
number = {2},
pages = {182--197},
year = {2002},
publisher = {IEEE}
}
@article{Arcuri2013,
abstract = {Many software engineering problems have been addressed with search algorithms. Search algorithms usually depend on several parameters (e.g., population size and crossover rate in genetic algorithms), and the choice of these parameters can have an impact on the performance of the algorithm. It has been formally proven in the No Free Lunch theorem that it is impossible to tune a search algorithm such that it will have optimal settings for all possible problems. So, how to properly set the parameters of a search algorithm for a given software engineering problem? In this paper, we carry out the largest empirical analysis so far on parameter tuning in search-based software engineering. More than one million experiments were carried out and statistically analyzed in the context of test data generation for object-oriented software using the EvoSuite tool. Results show that tuning does indeed have impact on the performance of a search algorithm. But, at least in the context of test data generation, it does not seem easy to find good settings that significantly outperform the "default" values suggested in the literature. This has very practical value for both researchers (e.g., when different techniques are compared) and practitioners. Using "default" values is a reasonable and justified choice, whereas parameter tuning is a long and expensive process that might or might not pay off in the end.},
author = {Arcuri, Andrea and Fraser, Gordon},
doi = {10.1007/s10664-013-9249-9},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Arcuri, Fraser - 2013 - Parameter tuning or default values An empirical investigation in search-based software engineering.pdf:pdf},
isbn = {1382-3256},
issn = {1382-3256},
journal = {Empirical Software Engineering},
keywords = {Design of experiments,EvoSuite,Java,Object-oriented,Response surface,Search-based software engineering,Test data generation,Tuning,Unit testing},
month = {jun},
number = {3},
pages = {594--623},
title = {{Parameter tuning or default values? An empirical investigation in search-based software engineering}},
volume = {18},
year = {2013}
}
@article{Alexander2004,
author = {Alexander, Roger T and Offutt, Jeff},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Alexander, Offutt - 2004 - Coupling-based Testing of O-O Programs.pdf:pdf},
journal = {Journal of Universal Computer Science},
keywords = {coverage testing,object-oriented software},
mendeley-groups = {Software Testing/Integration Testing},
number = {4},
pages = {391--427},
title = {{Coupling-based Testing of O-O Programs}},
volume = {10},
year = {2004}
}
@inproceedings{b2016learning,
title = {A learning-to-rank based fault localization approach using likely invariants},
author = {B Le, Tien-Duy and Lo, David and Le Goues, Claire and Grunske, Lars},
booktitle = {Proceedings of the 25th International Symposium on Software Testing and Analysis},
pages = {177--188},
year = {2016},
organization = {ACM}
}
@book{borba2010testing,
title = {Testing techniques in software engineering: Second pernambuco summer school on software engineering, PSSE 2007, Recife, Brazil, December 3-7, 2007, Revised Lectures},
author = {Borba, Paulo and Cavalcanti, Ana and Sampaio, Augusto and Woodcook, Jim},
volume = {6153},
year = {2010},
publisher = {Springer}
}
@inproceedings{GordonFlag,
title = {Combining search-based and constraint-based testing},
author = {Malburg, Jan and Fraser, Gordon},
booktitle = {Automated Software Engineering (ASE), 2011 26th IEEE/ACM International Conference on},
pages = {436--439},
year = {2011},
organization = {IEEE},
publisher = {{IEEE} Computer Society},
address = {Lawrence, KS, USA}
}
@inproceedings{Waheed,
author = {Waheed, Shahzada Zeeshan and Qamar, Usman},
booktitle = {2015 6th IEEE International Conference on Software Engineering and Service Science (ICSESS)},
doi = {10.1109/ICSESS.2015.7339088},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Waheed, Qamar - 2015 - Data flow based test case generation algorithm for object oriented integration testing.pdf:pdf},
isbn = {978-1-4799-8352-0},
keywords = {-path coverage,analysis,du-,object oriented testing,sut,test cases,test criteria},
mendeley-groups = {Software Testing/Integration Testing},
month = {sep},
pages = {423--427},
publisher = {IEEE},
title = {{Data flow based test case generation algorithm for object oriented integration testing}},
year = {2015}
}
@inproceedings{mondal2015,
author = {Mondal, Debajyoti and Hemmati, Hadi and Durocher, Stephane},
booktitle = {2015 IEEE 8th International Conference on Software Testing, Verification and Validation (ICST)},
doi = {10.1109/ICST.2015.7102588},
isbn = {978-1-4799-7125-1},
month = {apr},
pages = {1--10},
publisher = {IEEE},
series = {ICST '15},
title = {{Exploring test suite diversification and code coverage in multi-objective test case selection}},
year = {2015}
}
@article{fisher1936use,
author = {Fisher, Ronald A},
title = {The use of multiple measurements in taxonomic problems},
journal = {Annals of Eugenics},
volume = {7},
number = {2},
pages = {179-188},
doi = {10.1111/j.1469-1809.1936.tb02137.x},
year = {1936},
publisher = {Wiley Online Library}
}
@inproceedings{Tonella2012,
author = {Tonella, Paolo and Marchetto, Alessandro and Nguyen, Cu Duy and Jia, Yue and Lakhotia, Kiran and Harman, Mark},
doi = {10.1109/ICST.2012.82},
isbn = {9780769546704},
issn = {2159-4848},
booktitle = {Proceedings of the 5th IEEE International Conference on Software Testing, Verification and Validation, ICST 2012},
pages = {21--30},
publisher = {IEEE},
title = {{Finding the optimal balance between over and under approximation of models inferred from execution logs}},
year = {2012}
}
@inproceedings{McMinn2012,
author = {McMinn, Phil and Shahbaz, Muzammil and Stevenson, Mark},
booktitle = {IEEE Fifth International Conference on Software Testing, Verification and Validation (ICST '12)},
doi = {10.1109/ICST.2012.94},
isbn = {978-0-7695-4670-4},
issn = {2159-4848},
month = {apr},
pages = {141--150},
publisher = {IEEE},
title = {{Search-Based Test Input Generation for String Data Types Using the Results of Web Queries}},
year = {2012}
}
@inproceedings{BPT17concrash,
author = {Bianchi, Francesco A. and Pezz{\`{e}}, Mauro and Terragni, Valerio},
booktitle = {Proceedings of the 2017 11th Joint Meeting on Foundations of Software Engineering - ESEC/FSE 2017},
doi = {10.1145/3106237.3106292},
isbn = {9781450351058},
pages = {705--716},
publisher = {ACM Press},
title = {{Reproducing concurrency failures from crash stacks}},
year = {2017}
}
@article{Prowell2004,
author = {Prowell, S.J and Poore, J.H},
doi = {10.1016/S0164-1212(03)00241-3},
isbn = {0164-1212},
issn = {01641212},
journal = {Journal of Systems and Software},
month = {oct},
number = {2},
pages = {219--225},
title = {{Computing system reliability using Markov chain usage models}},
volume = {73},
year = {2004}
}
@article{Offutt2008,
author = {Offutt, Jeff and Abdurazik, Aynur and Schach, Stephen R.},
doi = {10.1007/s11219-008-9051-x},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Offutt, Abdurazik, Schach - 2008 - Quantitatively measuring object-oriented couplings.pdf:pdf},
issn = {0963-9314},
journal = {Software Quality Journal},
keywords = {abdurazik,analysis and evaluation {\'{a}},analysis {\'{a}} complexity measures,department,execution,george mason university,information and software engineering,j,object-oriented programming {\'{a}} quality,offutt,{\'{a}} a,{\'{a}} maintainability {\'{a}} test},
mendeley-groups = {Software Testing/Integration Testing},
month = {dec},
number = {4},
pages = {489--512},
title = {{Quantitatively measuring object-oriented couplings}},
volume = {16},
year = {2008}
}
@article{gay2015risks,
title = {The risks of coverage-directed test case generation},
author = {Gay, Gregory and Staats, Matt and Whalen, Michael and Heimdahl, Mats PE},
journal = {IEEE Transactions on Software Engineering},
volume = {41},
number = {8},
pages = {803--819},
year = {2015},
publisher = {IEEE}
}
@article{Arcuri2014,
abstract = {Randomized algorithms are widely used to address many types of software engineering problems, especially in the area of software verification and validation with a strong emphasis on test automation. However, randomized algorithms are affected by chance and so require the use of appropriate statistical tests to be properly analysed in a sound manner. This paper features a systematic review regarding recent publications in 2009 and 2010 showing that, overall, empirical analyses involving randomized algorithms in software engineering tend to not properly account for the random nature of these algorithms. Many of the novel techniques presented clearly appear promising, but the lack of soundness in their empirical evaluations casts unfortunate doubts on their actual usefulness. In software engineering, although there are guidelines on how to carry out empirical analyses involving human subjects, those guidelines are not directly and fully applicable to randomized algorithms. Furthermore, many of the textbooks on statistical analysis are written from the viewpoints of social and natural sciences, which present different challenges from randomized algorithms. To address the questionable overall quality of the empirical analyses reported in the systematic review, this paper provides guidelines on how to carry out and properly analyse randomized algorithms applied to solve software engineering tasks, with a particular focus on software testing, which is by far the most frequent application area of randomized algorithms within software engineering. Copyright {\textcopyright} 2012 John Wiley {\&} Sons, Ltd.},
author = {Arcuri, Andrea and Briand, Lionel},
doi = {10.1002/stvr.1486},
issn = {1099-1689},
journal = {Software Testing, Verification and Reliability},
keywords = {Bonferroni adjustment,bonferroni adjustment,confidence interval,effect size,nonparametric test,parametric test,statistical difference,survey,systematic review},
number = {3},
pages = {219--250},
title = {{A hitchhiker's guide to statistical tests for assessing randomized algorithms in software engineering}},
volume = {24},
year = {2014}
}
@inproceedings{lu2016does,
title = {How does regression test prioritization perform in real-world software evolution?},
author = {Lu, Yafeng and Lou, Yiling and Cheng, Shiyang and Zhang, Lingming and Hao, Dan and Zhou, Yangfan and Zhang, Lu},
booktitle = {2016 IEEE/ACM 38th International Conference on Software Engineering (ICSE)},
pages = {535--546},
year = {2016},
organization = {IEEE}
}
@software{evers_bjorn_2020_3897513,
author = {Evers, Björn and
Derakhshanfar, Pouria and
Devroey, Xavier and
Zaidman, Andy},
title = {{Unit test generation for common and uncommon
behaviors: replication package}},
month = jun,
year = 2020,
publisher = {Zenodo},
version = {v1.1.0},
doi = {10.5281/zenodo.3897513}
}
@inproceedings{MarkFlagRemoval,
title = {Improving evolutionary testing by flag removal},
author = {Harman, Mark and Hu, Lin and Hierons, Robert and Baresel, Andr{\'e} and Sthamer, Harmen},
booktitle = {Proceedings of the 4th Annual Conference on Genetic and Evolutionary Computation},
pages = {1359--1366},
year = {2002},
organization = {Morgan Kaufmann Publishers Inc.},
publisher = {Morgan Kaufmann},
address = {New York, USA}
}
@book{Miettinen1999,
author = {Miettinen, Kaisa},
title = {Nonlinear Multiobjective Optimization: Kaisa Miettinen},
year = {1999},
isbn = {978-1-4615-5563-6},
edition = {1st},
publisher = {Springer US}
}
@article{Cartaxo2011,
author = {Cartaxo, Emanuela G and Machado, Patr{\'{i}}cia D L and Neto, Francisco G Oliveira},
doi = {10.1002/stvr.413},
issn = {1099-1689},
journal = {Software Testing, Verification and Reliability},
number = {2},
pages = {75--100},
publisher = {John Wiley {\&} Sons, Ltd.},
title = {{On the use of a similarity function for test case selection in the context of model-based testing}},
volume = {21},
year = {2011}
}
@inproceedings{BellerICSE2018,
author = {Moritz Beller and Niels Spruit and Diomidis Spinellis and Andy Zaidman},
title = {On The Dichotomy of Debugging Behavior Among Programmers},
booktitle = {Proceedings of the International Conference on Software Engineering (ICSE)},
pages = {572--583},
publisher = {ACM},
year = {2018}
}
@inproceedings{Fraser2012,
author = {Fraser, Gordon and Arcuri, Andrea},
booktitle = {2012 IEEE Fifth International Conference on Software Testing, Verification and Validation},
doi = {10.1109/ICST.2012.92},
isbn = {978-0-7695-4670-4},
month = {apr},
pages = {121--130},
publisher = {IEEE},
title = {{The Seed is Strong: Seeding Strategies in Search-Based Software Testing}},
year = {2012}
}
@article{Panichella2016,
abstract = {{\textcopyright} 2016 ACM. Automated test generation tools have been widely investigated with the goal of reducing the cost of testing activities. However, generated tests have been shown not to help developers in detecting and finding more bugs even though they reach higher structural coverage compared to manual testing. The main reason is that generated tests are diff-cult to understand and maintain. Our paper proposes an approach, coined TestDescriber, which automatically generates test case summaries of the portion of code exercised by each individual test, thereby improving understandability. We argue that this approach can complement the current techniques around automated unit test generation or searchbased techniques designed to generate a possibly minimal set of test cases. In evaluating our approach we found that (1) developers find twice as many bugs, and (2) test case summaries significantly improve the comprehensibility of test cases, which is considered particularly useful by developers.},
author = {Panichella, Sebastiano and Panichella, Annibale and Beller, Moritz and Zaidman, Andy and Gall, Harald C.},
doi = {10.1145/2884781.2884847},
file = {:Users/pooria/Downloads/07886933.pdf:pdf},
isbn = {9781450339001},
issn = {02705257},
journal = {Proceedings - International Conference on Software Engineering},
keywords = {Empirical Study,Software testing,Test Case Summarization},
pages = {547--558},
publisher = {ACM},
title = {{The impact of test case summaries on bug fixing performance: An empirical investigation}},
volume = {14-22-May-2016},
year = {2016}
}
@article{Abdurazik2009,
author = {Abdurazik, Aynur and Offutt, Jeff},
doi = {10.1093/comjnl/bxm054},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Abdurazik, Offutt - 2009 - Using Coupling-Based Weights for the Class Integration and Test Order Problem.pdf:pdf},
issn = {0010-4620},
journal = {The Computer Journal},
keywords = {Class integration and test order,Coupling,OO testing},
mendeley-groups = {Software Testing/Integration Testing},
month = {aug},
number = {5},
pages = {557--570},
title = {{Using Coupling-Based Weights for the Class Integration and Test Order Problem}},
volume = {52},
year = {2009}
}
@book{Utting2007,
author = {Utting, Mark and Legeard, Bruno},
isbn = {0123725011 9780080466484},
publisher = {Morgan Kaufmann},
title = {{Practical Model-Based Testing: A Tools Approach}},
year = {2007}
}
@inproceedings{almasi2017industrial,
abstract = {—Automated unit test generation has been extensively studied in the literature in recent years. Previous studies on open source systems have shown that test generation tools are quite effective at detecting faults, but how effective and applicable are they in an industrial application? In this paper, we investigate this question using a life insurance and pension products calculator engine owned by SEB Life {\&} Pension Holding AB Riga Branch. To study fault-finding effectiveness, we extracted 25 real faults from the version history of this software project, and applied two up-to-date unit test generation tools for Java, EVOSUITE and RANDOOP, which implement search-based and feedback-directed random test generation, respectively. Automatically generated test suites detected up to 56.40{\%} (EVOSUITE) and 38.00{\%} (RANDOOP) of these faults. The analysis of our results demon-strates challenges that need to be addressed in order to improve fault detection in test generation tools. In particular, classification of the undetected faults shows that 97.62{\%} of them depend on either " specific primitive values " (50.00{\%}) or the construction of " complex state configuration of objects " (47.62{\%}). To study applicability, we surveyed the developers of the application under test on their experience and opinions about the test generation tools and the generated test cases. This leads to insights on requirements for academic prototypes for successful technology transfer from academic research to industrial practice, such as a need to integrate with popular build tools, and to improve the readability of the generated tests.},
author = {Almasi, M. Moein and Hemmati, Hadi and Fraser, Gordon and Arcuri, Andrea and Benefelds, Janis},
booktitle = {2017 IEEE/ACM 39th International Conference on Software Engineering: Software Engineering in Practice Track (ICSE-SEIP)},
doi = {10.1109/ICSE-SEIP.2017.27},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Almasi et al. - 2017 - An industrial evaluation of unit test generation Finding real faults in a financial application.pdf:pdf},
isbn = {978-1-5386-2717-4},
keywords = {Automated Tests Generation,Empirical Software Engineering,Random Testing,Search-based Testing},
month = {may},
pages = {263--272},
publisher = {IEEE},
title = {{An industrial evaluation of unit test generation: Finding real faults in a financial application}},
year = {2017}
}
@inproceedings{Kifetew2013,
author = {Kifetew, Fitsum Meshesha and Jin, Wei and Tiella, Roberto and Orso, Alessandro and Tonella, Paolo},
title = {SBFR: A Search Based Approach for Reproducing Failures of Programs with Grammar Based Input},
booktitle = {Proceedings of the 28th IEEE/ACM International Conference on Automated Software Engineering},
series = {ASE'13},
year = {2013},
isbn = {978-1-4799-0215-6},
pages = {604--609},
url = {https://doi.org/10.1109/ASE.2013.6693120},
doi = {10.1109/ASE.2013.6693120},
publisher = {IEEE Press},
address = {Piscataway, NJ, USA}
}
@inproceedings{tonella04,
title = {Evolutionary testing of classes},
author = {Tonella, Paolo},
booktitle = {ACM SIGSOFT Software Engineering Notes},
volume = {29},
number = {4},
pages = {119--128},
year = {2004},
organization = {ACM}
}
@inproceedings{kifetew2013b,
title = {Orthogonal exploration of the search space in evolutionary test case generation},
author = {Kifetew, Fitsum M and Panichella, Annibale and De Lucia, Andrea and Oliveto, Rocco and Tonella, Paolo},
booktitle = {Proc. Int'l Symposium on Software Testing and Analysis (ISSTA)},
pages = {257--267},
year = {2013},
organization = {ACM}
}
@inproceedings{Leemans2018,
author = {Leemans, Maikel and van der Aalst, Wil M. P. and van den Brand, Mark G. J.},
booktitle = {2018 IEEE 25th International Conference on Software Analysis, Evolution and Reengineering (SANER)},
doi = {10.1109/SANER.2018.8330248},
isbn = {978-1-5386-4969-5},
month = {mar},
pages = {502--506},
publisher = {IEEE},
title = {{The Statechart Workbench: Enabling scalable software event log analysis using process mining}},
year = {2018}
}
@inproceedings{Yuan2014,
author = {Yuan, Ding and Luo, Yu and Zhuang, Xin and Rodrigues, Guilherme Renna and Zhao, Xu and Zhang, Yongle and Jain, Pranay U. and Stumm, Michael},
title = {Simple Testing Can Prevent Most Critical Failures: An Analysis of Production Failures in Distributed Data-intensive Systems},
booktitle = {Proceedings of the 11th USENIX Conference on Operating Systems Design and Implementation},
series = {OSDI'14},
year = {2014},
isbn = {978-1-931971-16-4},
pages = {249--265},
publisher = {USENIX Association},
address = {Berkeley, CA, USA}
}
@article{Herbold2017,
author = {Herbold, Steffen and Harms, Patrick and Grabowski, Jens},
doi = {10.1007/s10009-016-0437-y},
issn = {1433-2779},
journal = {International Journal on Software Tools for Technology Transfer},
month = {jun},
number = {3},
pages = {309--324},
publisher = {Springer},
title = {{Combining usage-based and model-based testing for service-oriented architectures in the industrial practice}},
volume = {19},
year = {2017}
}
@inproceedings{Fraser2011whole,
author = {Fraser, Gordon and Arcuri, Andrea},
title = {Evolutionary Generation of Whole Test Suites},
booktitle = {Proc. Int'l Conf. on Quality Software},
series = {QSIC},
year = {2011},
isbn = {978-0-7695-4468-7},
pages = {31--40},
numpages = {10},
-url = {https://doi.org/10.1109/QSIC.2011.19},
-doi = {10.1109/QSIC.2011.19},
acmid = {2060610},
publisher = {IEEE},
-address = {Washington, DC, USA},
keywords = {Search based software engineering, branch coverage, genetic algorithm, length}
}
@inproceedings{Soltani2018b,
address = {Montpellier, France},
author = {Soltani, Mozhan and Derakhshanfar, Pouria and Panichella, Annibale and Devroey, Xavier and Zaidman, Andy and van Deursen, Arie},
booktitle = {Symposium on Search-Based Software Engineering. SSBSE 2018.},
doi = {10.1007/978-3-319-99241-9_18},
editor = {Colanzi, Thelma Elita and McMinn, Phil},
isbn = {9783319992402},
issn = {16113349},
number = {731529},
pages = {325--340},
publisher = {Springer},
series = {LNCS},
title = {{Single-objective Versus Multi-objectivized Optimization for Evolutionary Crash Reproduction}},
volume = {11036},
year = {2018}
}
@inproceedings{rojas2015combining,
title = {Combining multiple coverage criteria in search-based unit test generation},
author = {Rojas, Jos{\'e} Miguel and Campos, Jos{\'e} and Vivanti, Mattia and Fraser, Gordon and Arcuri, Andrea},
booktitle = {International Symposium on Search Based Software Engineering},
pages = {93--108},
year = {2015},
organization = {Springer}
}
@article{Sampath2007,
author = {Sampath, Sreedevi and Sprenkle, Sara and Gibson, Emily and Pollock, Lori L and Greenwald, Amie Souter},
journal = {IEEE Transactions on Software Engineering},
mendeley-groups = {Software Testing/Web application testing},
number = {10},
pages = {643--658},
title = {{Applying Concept Analysis to User-Session-Based Testing of Web Applications}},
volume = {33},
year = {2007}
}
@inproceedings{Chen2018b,
author = {Chen, Tao and Li, Miqing and Yao, Xin},
booktitle = {Proceedings of the Genetic and Evolutionary Computation Conference on - GECCO '18},
doi = {10.1145/3205455.3205513},
pages = {1419--1426},
publisher = {ACM Press},
title = {{On the Effects of Seeding Strategies: A Case for Search-based Multi-Objective Service Composition}},
year = {2018}
}
@article{Padhye2019,
abstract = {Programs expecting structured inputs often consist of both a syntactic analysis stage, which parses raw input, and a semantic analysis stage, which conducts checks on the parsed input and executes the core logic of the program. Generator-based testing tools in the lineage of QuickCheck are a promising way to generate random syntactically valid test inputs for these programs. We present Zest, a technique which automatically guides QuickCheck-like randominput generators to better explore the semantic analysis stage of test programs. Zest converts random-input generators into deterministic parametric generators. We present the key insight that mutations in the untyped parameter domain map to structural mutations in the input domain. Zest leverages program feedback in the form of code coverage and input validity to perform feedback-directed parameter search. We evaluate Zest against AFL and QuickCheck on five Java programs: Maven, Ant, BCEL, Closure, and Rhino. Zest covers 1.03x-2.81x as many branches within the benchmarks semantic analysis stages as baseline techniques. Further, we find 10 new bugs in the semantic analysis stages of these benchmarks. Zest is the most effective technique in finding these bugs reliably and quickly, requiring at most 10 minutes on average to find each bug.},
author = {Padhye, Rohan and Lemieux, Caroline and Sen, Koushik and Papadakis, Mike and {Le Traon}, Yves},
doi = {10.1145/3293882.3330576},
file = {:Users/pooria/Downloads/issta19main-p272-p.pdf:pdf},
isbn = {9781450362245},
keywords = {Structure-aware fuzzing, property-based testing, random testing,acm reference format,and yves,caroline lemieux,koushik sen,mike papadakis,property-based testing,random testing,rohan padhye,structure-aware fuzzing},
pages = {329--340},
title = {{Semantic fuzzing with zest}},
year = {2019}
}
@inproceedings{McMinn2011,
author = {McMinn, Phil},
title = {Search-based software testing: Past, present and future},
booktitle = {Proceedings of the 2011 IEEE Fourth International Conference on Software Testing, Verification and Validation Workshops},
series = {ICSTW '11},
year = {2011},
isbn = {978-0-7695-4345-1},
pages = {153--163},
doi = {10.1109/ICSTW.2011.100},
publisher = {IEEE Computer Society},
address = {Washington, DC, USA},
keywords = {Search-Based Software Testing, Search-Based Software Engineering}
}
@misc{Vieira2016,
author = {Vieira, Mark},
title = {{Introducing Compile-Only Dependencies}},
howpublished = {\url{https://blog.gradle.org/introducing-compile-only-dependencies}},
year = {2016},
note = {[Online; accessed 25-January-2018]}
}
@inproceedings{braione2018sushi,
title = {SUSHI: a test generator for programs with complex structured inputs},
author = {Braione, Pietro and Denaro, Giovanni and Mattavelli, Andrea and Pezz{\`e}, Mauro},
booktitle = {2018 IEEE/ACM 40th International Conference on Software Engineering: Companion (ICSE-Companion)},
pages = {21--24},
year = {2018},
organization = {IEEE}
}
@inproceedings{arcuri2016java,
title = {Java enterprise edition support in search-based junit test generation},
author = {Arcuri, Andrea and Fraser, Gordon},
booktitle = {International Symposium on Search Based Software Engineering},
pages = {3--17},
year = {2016},
organization = {Springer}
}
@article{Kallepalli2001,
author = {Kallepalli, Chaitanya and Tian, Jeff},
doi = {10.1109/32.965342},
journal = {TSE},
month = {nov},
number = {11},
pages = {1023--1036},
title = {{Measuring and modeling usage and reliability for statistical Web testing}},
volume = {27},
year = {2001}
}
@inproceedings{pearson2017evaluating,
title = {Evaluating and improving fault localization},
author = {Pearson, Spencer and Campos, Jos{\'e} and Just, Ren{\'e} and Fraser, Gordon and Abreu, Rui and Ernst, Michael D and Pang, Deric and Keller, Benjamin},
booktitle = {Proceedings of the 39th International Conference on Software Engineering},
pages = {609--620},
year = {2017},
organization = {IEEE Press}
}
@inproceedings{Derakhshanfar2019b,
address = {Brussels},
author = {Derakhshanfar, Pouria and Devroey, Xavier and Panichella, Annibale and Zaidman, Andy and Deursen, Arie Van},
booktitle = {18th Belgium-Netherlands Software Evolution Workshop (BENEVOL '19)},
title = {{Presentation Abstract : Generating Class Integration Tests Using Call Site Information}},
url = {http://soft.vub.ac.be/benevol2019/},
year = {2019}
}
@incollection{Arcuri2011a,
author = {Arcuri, Andrea and Fraser, Gordon},
booktitle = {Population English Edition},
doi = {10.1007/978-3-642-23716-4_6},
isbn = {9781450305624},
pages = {33--47},
title = {{On Parameter Tuning in Search Based Software Engineering}},
year = {2011}
}
@article{Jaccard1901,
author = {Jaccard, Paul},
doi = {10.5169/seals-266450},
journal = {Bulletin del la Soci{\'{e}}t{\'{e}} Vaudoise des Sciences Naturelles},
pages = {547--579},
title = {{{\'{E}}tude comparative de la distribution florale dans une portion des Alpes et des Jura}},
volume = {37},
year = {1901}
}
@article{zhang2007moea,
title = {MOEA/D: A multiobjective evolutionary algorithm based on decomposition},
author = {Zhang, Qingfu and Li, Hui},
journal = {IEEE Transactions on evolutionary computation},
volume = {11},
number = {6},
pages = {712--731},
year = {2007},
publisher = {IEEE}
}
@inproceedings{feldt2008,
title = {Searching for cognitively diverse tests: Towards universal test diversity metrics},
author = {Feldt, Robert and Torkar, Richard and Gorschek, Tony and Afzal, Wasif},
booktitle = {Proc. Int'l Conf. Software Testing Verification and Validation Workshops (ICSTW)},
pages = {178--186},
year = {2008},
organization = {IEEE}
}
@inproceedings{Derakhshanfar2020a,
address = {Canc{\'{u}}n, Mexico},
author = {Derakhshanfar, Pouria and Devroey, Xavier and Zaidman, Andy and van Deursen, Arie and Panichella, Annibale},
booktitle = {Genetic and Evolutionary Computation Conference Companion (GECCO '20 Companion)},
doi = {10.1145/3377929.3390077},
publisher = {ACM},
title = {{Crash Reproduction Using Helper Objectives}},
year = {2020}
}
@inproceedings{Leitner2007,
author = {Leitner, Andreas and Ciupa, Ilinca and Oriol, Manuel and Meyer, Bertrand and Fiva, Arno},
title = {Contract Driven Development = Test Driven Development - Writing Test Cases},
booktitle = {Proceedings of the the 6th Joint Meeting of the European Software Engineering Conference and the ACM SIGSOFT Symposium on The Foundations of Software Engineering},
series = {ESEC-FSE '07},
year = {2007},
isbn = {978-1-59593-811-4},
location = {Dubrovnik, Croatia},
pages = {425--434},
doi = {10.1145/1287624.1287685},
publisher = {ACM},
address = {New York, NY, USA},
keywords = {unit testing}
}
@article{Arcuri2019,
abstract = {RESTful APIs are widespread in industry, especially in enterprise applications developed with a microservice architecture. A RESTful web service will provide data via an API over the network using HTTP, possibly interacting with databases and other web services. Testing a RESTful API poses challenges, because inputs/outputs are sequences of HTTP requests/responses to a remote server. Many approaches in the literature do black-box testing, because often the tested API is a remote service whose code is not available. In this article, we consider testing from the point of view of the developers, who have full access to the code that they are writing. Therefore, we propose a fully automated white-box testing approach, where test cases are automatically generated using an evolutionary algorithm. Tests are rewarded based on code coverage and fault-finding metrics. However, REST is not a protocol but rather a set of guidelines on how to design resources accessed over HTTP endpoints. For example, there are guidelines on how related resources should be structured with hierarchical URIs and how the different HTTP verbs should be used to represent well-defined actions on those resources. Test-case generation for RESTful APIs that only rely on white-box information of the source code might not be able to identify how to create prerequisite resources needed before being able to test some of the REST endpoints. Smart sampling techniques that exploit the knowledge of best practices in RESTful API design are needed to generate tests with predefined structures to speed up the search. We implemented our technique in a tool called EvoMaster, which is open source. Experiments on five open-source, yet non-trivial, RESTful services show that our novel technique automatically found 80 real bugs in those applications. However, obtained code coverage is lower than the one achieved by the manually written test suites already existing in those services. Research directions on how to further improve such an approach are therefore discussed, such as the handling of SQL databases.},
author = {Arcuri, Andrea},
doi = {10.1145/3293455},
file = {:Users/pooria/Downloads/a3-arcuri.pdf:pdf},
issn = {15577392},
journal = {ACM Transactions on Software Engineering and Methodology},
keywords = {REST,Software engineering,Testing,Web service},
number = {1},
pages = {1--37},
title = {{RESTful API automated test case generation with Evomaster}},
volume = {28},
year = {2019}
}
@article{Panichella2017c,
author = {Panichella, Annibale and Molina, Urko Rueda},
doi = {10.1109/SBST.2017.7},
isbn = {9781538627891},
journal = {Proceedings - 2017 IEEE/ACM 10th International Workshop on Search-Based Software Testing, SBST 2017},
pages = {32--38},
title = {{Java unit testing tool competition - Fifth round}},
year = {2017}
}
@inproceedings{xiao2011precise,
title = {Precise identification of problems for structural test generation},
author = {Xiao, Xusheng and Xie, Tao and Tillmann, Nikolai and De Halleux, Jonathan},
booktitle = {Software Engineering (ICSE), 2011 33rd International Conference on},
pages = {611--620},
year = {2011},
organization = {IEEE},
publisher = {{ACM}},
address = {Waikiki, Honolulu , HI, USA}
}
@misc{JDK7024096,
author = {JDK},
title = {{Stack trace has invalid line numbers}},
howpublished = {\url{https://bugs.openjdk.java.net/browse/JDK-7024096}},
year = {2016},
note = {[Online; accessed 25-January-2018]}
}
@inproceedings{Filieri:2011:REP:1985793.1985840,
author = {Filieri, Antonio and Ghezzi, Carlo and Tamburrelli, Giordano},
title = {Run-time Efficient Probabilistic Model Checking},
booktitle = {Proceedings of the 33rd International Conference on Software Engineering},
series = {ICSE '11},
year = {2011},
isbn = {978-1-4503-0445-0},
location = {Waikiki, Honolulu, HI, USA},
pages = {341--350},
numpages = {10},
doi = {10.1145/1985793.1985840},
acmid = {1985840},
publisher = {ACM}
}
@article{Menzies2013,
author = {T. {Menzies} and T. {Zimmermann}},
journal = {IEEE Software},
title = {Software Analytics: So What?},
year = {2013},
volume = {30},
number = {4},
pages = {31-37}
}
@inproceedings{Bansal2009,
author = {Bansal, Priti and Sabharwal, Sangeeta and Sidhu, Pameeta},
booktitle = {2009 Proceeding of International Conference on Methods and Models in Computer Science (ICM2CS)},
doi = {10.1109/ICM2CS.2009.5397936},
file = {:Users/xavierdevroey/Dropbox/These/Articles/Bansal, Sabharwal, Sidhu - 2009 - An investigation of strategies for finding test order during Integration testing of object Oriented ap.pdf:pdf},
isbn = {978-1-4244-5051-0},
keywords = {coupling,integration testing,object,oriented applications,test order},
mendeley-groups = {Software Testing/Integration Testing},
month = {dec},
pages = {1--8},
publisher = {IEEE},
title = {{An investigation of strategies for finding test order during Integration testing of object Oriented applications}},
year = {2009}