references.bib

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% This file is automatically generated.  Do not modify this
% file directly, or else your changes will be overwritten.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%"
@article{ferguson73dp,
  author = {T. S. Ferguson},
  journal = {Annals of Statistics},
  pages = {209--230},
  title = {A {B}ayesian Analysis of Some Nonparametric Problems},
  volume = {1},
  year = {1973},
}
@article{ferguson74prior,
  author = {T. S. Ferguson},
  journal = {Annals of Statistics},
  pages = {615--629},
  title = {Prior distributions on spaces of probability measures},
  volume = {2},
  year = {1974},
}
@article{blackwell73urn,
  author = {D. Blackwell and J. B. MacQueen},
  journal = {Annals of Statistics},
  pages = {353--355},
  title = {{F}erguson Distributions via {P}\'olya Urn Schemes},
  volume = {1},
  year = {1973},
}
@article{aldous85exch,
  author = {D. Aldous},
  journal = {Springer Lecture Notes in Math},
  pages = {1--198},
  title = {Exchangeability and related topics},
  volume = {1117},
  year = {1985},
}
@article{antoniak74dpmix,
  author = {C. E. Antoniak},
  journal = {Annals of Statistics},
  pages = {1152--1174},
  title = {Mixtures of {D}irichlet Processes with Applications to {B}ayesian Nonparametric Problems},
  volume = {2},
  year = {1974},
}
@article{sethuraman94stick,
  author = {J. Sethuraman},
  journal = {Statistica Sinica},
  pages = {639--650},
  title = {A Constructive Definition of {D}irichlet Priors},
  volume = {4},
  year = {1994},
}
@article{escobar95mixture,
  author = {M. D. Escobar and M. West},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {577--588},
  title = {{B}ayesian Density Estimation and Inference Using Mixtures},
  volume = {90},
  year = {1995},
}
@techreport{west95hyper,
  author = {M. West},
  institution = {Department of Statistics, Duke University (Duke University)},
  title = {Hyperparameter estimation in {D}irichlet process mixture models},
  year = {1995},
}
@article{pitman97yor,
  author = {J. Pitman and M. Yor},
  journal = {Annals of Probability},
  pages = {855--900},
  title = {The two-parameter {P}oisson-{D}irichlet distribution derived from a stable subordinator},
  volume = {25},
  year = {1997},
}
@article{ishwaran02exact,
  author = {H. Ishwaran and M. Zarepour},
  journal = {Canadian Journal of Statististics},
  pages = {269--284},
  title = {Exact and approximate sum-representations for the {D}irichlet process},
  volume = {30},
  year = {2002},
}
@inproceedings{griffiths06ibp,
  author = {T. Griffiths and Z. Ghahramani},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Infinite Latent Feature Models and the {I}ndian Buffet Process},
  year = {2006},
}
@inproceedings{thibaux07hbp,
  author = {R. Thibaux and M. I. Jordan},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {Hierarchical {B}eta Processes and the {I}ndian Buffet Process},
  year = {2007},
}
@inproceedings{xing08haplotype,
  author = {J. V. Gael and Y. Saatci and Y. W. Teh and Z. Ghahramani},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {1088--1095},
  title = {Beam Sampling for the Infinite Hidden {M}arkov Model},
  year = {2008},
}
@inproceedings{xing04haplotype,
  author = {E. P. Xing and R. Sharan and M. I. Jordan},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {111--118},
  title = {{B}ayesian Haplotype Inference via the {D}irichlet Process},
  year = {2004},
}
@article{teh06hdp,
  author = {Y. W. Teh and M. I. Jordan and M. Beal and D. Blei},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {1566--1581},
  title = {Hierarchical {D}irichlet processes},
  volume = {101},
  year = {2006},
}
@article{rodriguez08ndp,
  author = {A. Rodriguez and D. B. Dunson and A. E. Gelfand},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {1131--1144},
  title = {The nested {D}irichlet process},
  volume = {103},
  year = {2008},
}
@inproceedings{beal02ihmm,
  author = {M. Beal and Z. Ghahramani and C. Rasmussen},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {577--584},
  title = {The infinite hidden {M}arkov model},
  year = {2002},
}
@inproceedings{sohn07ihmm,
  author = {K. Sohn and E. P. Xing},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {A Hidden {M}arkov {D}irichlet Process Model for Genetic Recombination in Open Ancestral Space},
  year = {2007},
}
@inproceedings{teh06pitmanyor,
  author = {Y. W. Teh},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  pages = {985--992},
  title = {A Hierarchical {B}ayesian Language Model Based On {P}itman-{Y}or Processes},
  year = {2006},
}
@inproceedings{goldwater06segmentation,
  author = {S. Goldwater and T. Griffiths and M. Johnson},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {Contextual Dependencies in Unsupervised Word Segmentation},
  year = {2006},
}
@inproceedings{sudderth06transformed,
  author = {E. B. Sudderth and A. B. Torralba and W. T. Freeman and A. S. Willsky},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {1297--1304},
  title = {Describing Visual Scenes using Transformed {D}irichlet Processes},
  year = {2006},
}
@article{daume05clustering,
  author = {H. Daume and D. Marcu},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {1--48},
  title = {A {B}ayesian Model for Supervised Clustering with the {D}irichlet Process Prior},
  volume = {1},
  year = {2005},
}
@article{blei05variational,
  author = {D. Blei and M. I. Jordan},
  journal = {Bayesian Analysis},
  pages = {121--144},
  title = {Variational Inference for {D}irichlet Process Mixtures},
  volume = {1},
  year = {2005},
}
@inproceedings{teh07collapsed,
  author = {Y. W. Teh and D. Newman and M. Welling},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {1353--1360},
  title = {A Collapsed Variational {B}ayesian Inference Algorithm for {L}atent {D}irichlet {A}llocation},
  year = {2007},
}
@inproceedings{kurihara07accelerated,
  author = {K. Kurihara and M. Welling and N. Vlassis},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Accelerated Variational {D}irichlet Mixture Models},
  year = {2007},
}
@inproceedings{kurihara07collapsed,
  author = {K. Kurihara and M. Welling and Y. W. Teh},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  title = {Collapsed Variational {D}irichlet Process Mixture Models},
  year = {2007},
}
@inproceedings{daume07astar,
  author = {H. Daume},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {Fast search for {D}irichlet process mixture models},
  year = {2007},
}
@inproceedings{heller05hierarchical,
  author = {K. A. Heller and Z. Ghahramani},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {{B}ayesian Hierarchical Clustering},
  year = {2005},
}
@article{ishwaran01gibbs,
  author = {H. Ishwaran and L. F. James},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {161--173},
  title = {{G}ibbs Sampling Methods for Stick-Breaking Priors},
  volume = {96},
  year = {2001},
}
@article{papaspiliopoulos08retro,
  author = {O. Papaspiliopoulos and G. O. Roberts},
  journal = {Biometrika},
  pages = {169--186},
  title = {Retrospective {MCMC} for {D}irichlet process hierarchical models},
  volume = {95},
  year = {2008},
}
@article{walker04slice,
  author = {S. G. Walker},
  journal = {Communications in Statistics - Simulation and Computation},
  pages = {45--54},
  title = {Sampling the {D}irichlet Mixture Model with Slices},
  volume = {36},
  year = {2004},
}
@techreport{dahl03splitmerge,
  author = {D. B. Dahl},
  institution = {Department of Statistics, University of Wisconsin (U. Wisconsin)},
  title = {An Improved Merge-Split Sampler for Conjugate {D}irichlet Process Mixture Models},
  year = {2003},
}
@techreport{dahl03univariate,
  author = {D. B. Dahl},
  institution = {Department of Statistics, University of Wisconsin (U. Wisconsin)},
  title = {Modal Clustering in a Univariate Class of Product Partition Models},
  year = {2003},
}
@techreport{neal00dp,
  author = {R. Neal},
  institution = {Department of Statistics, University of Toronto (U. Toronto)},
  title = {{M}arkov Chain Sampling Methods for {D}irichlet Process Mixture Models},
  year = {2000},
}
@techreport{jain00splitmerge,
  author = {S. Jain and R. Neal},
  institution = {Department of Statistics, University of Toronto (U. Toronto)},
  title = {A Split-Merge {M}arkov Chain {M}onte {C}arlo Procedure for the {D}irichlet Process Mixture Model},
  year = {2000},
}
@techreport{jain05splitmerge,
  author = {S. Jain and R. Neal},
  institution = {Department of Statistics, University of Toronto (U. Toronto)},
  title = {Splitting and merging components of a nonconjugate {D}irichlet process mixture model},
  year = {2005},
}
@article{blei03lda,
  author = {D. Blei and A. Ng and M. I. Jordan},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {993--1022},
  title = {Latent {D}irichlet Allocation},
  volume = {3},
  year = {2003},
}
@inproceedings{hamze04fields,
  author = {F. Hamze and N. d. Freitas},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {From Fields to Trees},
  year = {2004},
}
@inproceedings{welling06flexible,
  author = {M. Welling},
  booktitle = {International Conference on Machine Learning Workshop on Nonparametric Bayesian Methods},
  title = {Flexible Priors for Infinite Mixture Models},
  year = {2006 2006},
}
@article{johnson84randproj,
  author = {W. Johnson and J. Lindenstrauss},
  journal = {Contemporary Mathematics},
  pages = {189--206},
  title = {Extensions of {L}ipschitz maps into a {H}ilbert space},
  volume = {26},
  year = {1984},
}
@inproceedings{dasgupta99mixgauss,
  author = {S. Dasgupta},
  booktitle = {Foundations of Computer Science (FOCS)},
  title = {Learning mixtures of {G}aussians},
  year = {1999},
}
@article{kannan97convex,
  author = {R. Kannan and L. Lovasz and M. Simonovits},
  journal = {Random Structures and Algorithms},
  pages = {1--50},
  title = {Random walks and an ${O}^*(n^5)$ volume algorithm for convex bodies},
  volume = {11},
  year = {1997},
}
@inproceedings{friedman00bayesian,
  author = {N. Friedman and D. Koller},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  pages = {201--210},
  title = {Being {B}ayesian about {B}ayesian Network Structure: A {B}ayesian Approach to Structure Discovery in {B}ayesian Networks},
  year = {2000},
}
@article{swendsen87,
  author = {R. H. Swendsen and J. S. Wang},
  journal = {Physics Review Letters},
  pages = {86--88},
  title = {Nonuniversal critical dynamics in {MC} simulations},
  volume = {58},
  year = {1987},
}
@article{barbu05swendsen,
  author = {A. Barbu and S. C. Zhu},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {1239--1253},
  title = {Generalizing {S}wendsen-{W}ang to sampling arbitrary posterior probabilities},
  volume = {27},
  year = {2005},
}
@article{tanner87da,
  author = {M. A. Tanner and W. H. Wong},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {528--540},
  title = {The calculation of posterior distributions by data augmentation},
  volume = {82},
  year = {1987},
}
@article{liu98pxem,
  author = {C. Liu and D. Rubin and Y. N. Wu},
  journal = {Biometrika},
  pages = {755--770},
  title = {Parameter expansion to accelerate {EM}: the {PX-EM} algorithm},
  volume = {85},
  year = {1998},
}
@article{liu99pxda,
  author = {J. Liu and Y. Wu},
  journal = {Journal of the American Statistical Association (JASA)},
  pages = {1264--1274},
  title = {Parameter expansion for data augmentation},
  volume = {94},
  year = {1999},
}
@techreport{pitman02process,
  author = {J. Pitman},
  institution = {Department of Statistics, University of California at Berkeley (UC Berkeley)},
  number = {621},
  title = {Combinatorial Stochastic Processes},
  year = {2002},
}
@book{bertsekas99nlp,
  author = {D. Bertsekas},
  publisher = {Athena Scientific},
  title = {Nonlinear Programming},
  year = {1999},
}
@techreport{mackay97hmm,
  author = {D. MacKay},
  institution = {University of Cambridge},
  title = {Ensemble Learning for Hidden {M}arkov Models},
  year = {1997},
}
@inproceedings{hermjakob01parsing,
  author = {U. Hermjakob},
  booktitle = {Workshop on Open-domain question answering, ACL},
  pages = {1--6},
  title = {Parsing and Question Classification for Question Answering},
  year = {2001},
}
@article{gildea02semantic,
  author = {D. Gildea and D. Jurafsky},
  journal = {Computational Linguistics},
  pages = {245--288},
  title = {Automatic Labeling of Semantic Roles},
  volume = {28},
  year = {2002},
}
@inproceedings{galley04translation,
  author = {M. Galley and M. Hopkins and K. Knight and D. Marcu},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {273--280},
  title = {What's in a translation rule?},
  year = {2004},
}
@inproceedings{melamed04smt,
  author = {I. D. Melamed},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Statistical Machine Translation by Parsing},
  year = {2004},
}
@article{chomsky56models,
  author = {N. Chomsky},
  journal = {IRE Transactions on Information Theory},
  pages = {113--124},
  title = {Three models for the description of language},
  volume = {2},
  year = {1956},
}
@book{manning99nlp,
  author = {C. Manning and H. Schütze},
  publisher = {MIT Press},
  title = {Foundations of Statistical Natural Language Processing},
  year = {1999},
}
@article{marcus93treebank,
  author = {M. P. Marcus and M. A. Marcinkiewicz and B. Santorini},
  journal = {Computational Linguistics},
  pages = {313--330},
  title = {Building a large annotated corpus of {E}nglish: the {P}enn {T}reebank},
  volume = {19},
  year = {1993},
}
@inproceedings{charniak96treebank,
  author = {E. Charniak},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {1031--1036},
  title = {Tree-bank Grammars},
  year = {1996},
}
@inproceedings{kurihara04varpcfg,
  author = {K. Kurihara and T. Sato},
  booktitle = {International Joint Conference on Natural Language Processing Workshop Beyond Shallow Analyses},
  title = {An Application of the Variational {B}ayesian Approach to Probabilistic Context-Free Grammars},
  year = {2004},
}
@inproceedings{kurihara06varinduct,
  author = {K. Kurihara and T. Sato},
  booktitle = {International Colloquium on Grammatical Inference},
  title = {Variational {B}ayesian Grammar Induction for Natural Language},
  year = {2006},
}
@inproceedings{petrov06latent,
  author = {S. Petrov and L. Barrett and R. Thibaux and D. Klein},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  pages = {433--440},
  title = {Learning Accurate, Compact, and Interpretable Tree Annotation},
  year = {2006},
}
@inproceedings{stolcke94merge,
  author = {A. Stolcke and S. Omohundro},
  booktitle = {International Colloquium on Grammatical Inference and Applications},
  pages = {106--118},
  title = {Inducing Probabilistic Grammars by {B}ayesian Model Merging},
  year = {1994},
}
@inproceedings{wolff88syntax,
  author = {J. G. Wolff},
  booktitle = {Categories and processes in language acquisition},
  pages = {179--215},
  title = {Learning syntax and meanings through optimization and distributional analysis},
  year = {1988},
}
@phdthesis{collins99thesis,
  author = {M. Collins},
  school = {University of Pennsylvania},
  title = {Head-Driven Statistical Models for Natural Language Parsing},
  year = {1999},
}
@inproceedings{eisner96dependency,
  author = {J. Eisner},
  booktitle = {International Conference on Computational Linguistics (COLING)},
  pages = {340--345},
  title = {Three New Probabilistic Models for Dependency Parsing: An Exploration},
  year = {1996},
}
@inproceedings{charniak00maxent,
  author = {E. Charniak},
  booktitle = {Applied Natural Language Processing and North American Association for Computational Linguistics (ANLP/NAACL)},
  pages = {132--139},
  title = {A maximum-entropy-inspired parser},
  year = {2000},
}
@inproceedings{klein03unlexicalized,
  author = {D. Klein and C. Manning},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {423--430},
  title = {Accurate Unlexicalized Parsing},
  year = {2003},
}
@article{johnson98parent,
  author = {M. Johnson},
  journal = {Computational Linguistics},
  pages = {613--632},
  title = {{PCFG} Models of Linguistic Tree Representations},
  volume = {24},
  year = {1998},
}
@inproceedings{matsuzaki05latent,
  author = {T. Matsuzaki and Y. Miyao and J. Tsujii},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {75--82},
  title = {Probabilistic {CFG} with Latent Annotations},
  year = {2005},
}
@inproceedings{johnson06adaptor,
  author = {M. Johnson and T. Griffiths and S. Goldwater},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {641--648},
  title = {Adaptor Grammars: A Framework for Specifying Compositional Nonparametric {B}ayesian Models},
  year = {2006},
}
@inproceedings{blei04nested,
  author = {D. Blei and T. Griffiths and M. I. Jordan and J. Tenenbaum},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Hierarchical topic models and the nested {C}hinese restaurant process},
  year = {2004},
}
@inproceedings{goldwater05interpolate,
  author = {S. Goldwater and T. Griffiths and M. Johnson},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Interpolating Between Types and Tokens by Estimating Power-Law Generators},
  year = {2005},
}
@inproceedings{johnson07mcmc,
  author = {M. Johnson and T. Griffiths and S. Goldwater},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {139--146},
  title = {{B}ayesian Inference for {PCFG}s via {M}arkov Chain {M}onte {C}arlo},
  year = {2007},
}
@inproceedings{haghighi07coref,
  author = {A. Haghighi and D. Klein},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Unsupervised Coreference Resolution in a Nonparametric {B}ayesian Model},
  year = {2007},
}
@inproceedings{daume06sum,
  author = {H. Daume and D. Marcu},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {{B}ayesian Query-Focused Summarization},
  year = {2006},
}
@inproceedings{finkel07inftree,
  author = {J. R. Finkel and T. Grenager and C. Manning},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {272--279},
  title = {The Infinite Tree},
  year = {2007},
}
@inproceedings{petrov07split,
  author = {S. Petrov and D. Klein},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {404--411},
  title = {Learning and Inference for Hierarchically Split {PCFG}s},
  year = {2007},
}
@inproceedings{mcclosky06self,
  author = {D. McClosky and E. Charniak and M. Johnson},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {Reranking and Self-Training for Parser Adaptation},
  year = {2006},
}
@article{lari90scfg,
  author = {K. Lari and S. J. Young},
  journal = {Computer Speech and Language},
  pages = {35--56},
  title = {The estimation of stochastic context-free grammars using the inside-outside algorithm},
  volume = {4},
  year = {1990},
}
@article{brown93mt,
  author = {P. F. Brown and S. A. D. Pietra and V. J. D. Pietra and R. L. Mercer},
  journal = {Computational Linguistics},
  pages = {263--311},
  title = {The Mathematics of Statistical Machine Translation: Parameter Estimation},
  volume = {19},
  year = {1993},
}
@inproceedings{neal98gem,
  author = {R. M. Neal and G. E. Hinton},
  booktitle = {Learning in Graphical Models},
  pages = {355--368},
  title = {A new view of the {EM} algorithm that justifies incremental, sparse and other variants},
  year = {1998},
}
@inproceedings{collins99unsupervised,
  author = {M. Collins and Y. Singer},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Unsupervised Models for Named Entity Classification},
  year = {1999},
}
@inproceedings{chen96smoothing,
  author = {S. F. Chen and J. Goodman},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {An Empirical Study of Smoothing Techniques for Language Modeling},
  year = {1996},
}
@inproceedings{ney96hmm,
  author = {H. Ney and S. Vogel},
  booktitle = {International Conference on Computational Linguistics (COLING)},
  pages = {836--841},
  title = {{HMM}-Based Word Alignment in Statistical Translation},
  year = {1996},
}
@inproceedings{blum98cotraining,
  author = {A. Blum and T. Mitchell},
  booktitle = {Conference on Learning Theory (COLT)},
  title = {Combining Labeled and Unlabeled Data with Co-training},
  year = {1998},
}
@inproceedings{carroll92dependency,
  author = {G. Carroll and E. Charniak},
  booktitle = {Workshop Notes for Statistically-Based NLP Techniques, AAAI},
  pages = {1--13},
  title = {Two Experiments on Learning Probabilistic Dependency Grammars from Corpora},
  year = {1992},
}
@inproceedings{haghighi06induction,
  author = {A. Haghighi and D. Klein},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {Prototype-based Grammar Induction},
  year = {2006},
}
@inproceedings{pereira92bracket,
  author = {F. Pereira and Y. Shabes},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {128--135},
  title = {Inside-outside reestimation from partially bracketed corpora},
  year = {1992},
}
@article{merialdo94tagging,
  author = {B. Merialdo},
  journal = {Computational Linguistics},
  pages = {155--171},
  title = {Tagging {E}nglish text with a probabilistic model},
  volume = {20},
  year = {1994},
}
@inproceedings{clark01induction,
  author = {A. Clark},
  booktitle = {Computational Natural Language Learning (CoNLL)},
  title = {Unsupervised induction of stochastic context free grammars with distributional clustering},
  year = {2001},
}
@inproceedings{paskin02bigrams,
  author = {M. A. Paskin},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Grammatical Bigrams},
  year = {2002},
}
@inproceedings{klein04induction,
  author = {D. Klein and C. D. Manning},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {478--485},
  title = {Corpus-Based Induction of Syntactic Structure: Models of Dependency and Constituency},
  year = {2004},
}
@inproceedings{smith05contrastive,
  author = {N. Smith and J. Eisner},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {354--362},
  title = {Contrastive Estimation: Training Log-Linear Models on Unlabeled Data},
  year = {2005},
}
@inproceedings{smith06anneal,
  author = {N. Smith and J. Eisner},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {Annealing Structural Bias in Multilingual Weighted Grammar Induction},
  year = {2006},
}
@inproceedings{goldwater07pos,
  author = {S. Goldwater and T. Griffiths},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {A Fully {B}ayesian Approach to Unsupervised Part-of-Speech Tagging},
  year = {2007},
}
@inproceedings{johnson08synergy,
  author = {M. Johnson},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  pages = {398--406},
  title = {Using Adaptor Grammars to Identify Synergies in the Unsupervised Acquisition of Linguistic Structure},
  year = {2008},
}
@inproceedings{johnson07pos,
  author = {M. Johnson},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  title = {Why doesn't {EM} find good {HMM} {POS}-taggers?},
  year = {2007},
}
@article{goldwater09segmentation,
  author = {S. Goldwater and T. Griffiths and M. Johnson},
  journal = {Cognition},
  pages = {21--54},
  title = {A {B}ayesian Framework for Word Segmentation: Exploring the Effects of Context},
  volume = {112},
  year = {2009},
}
@inproceedings{johnson09segmentation,
  author = {M. Johnson and S. Goldwater},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {317--325},
  title = {Improving nonparameteric {B}ayesian inference: experiments on unsupervised word segmentation with adaptor grammars},
  year = {2009},
}
@inproceedings{cohn09ptsg,
  author = {T. Cohn and S. Goldwater and P. Blunsom},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {548--556},
  title = {Inducing Compact but Accurate Tree-Substitution Grammars},
  year = {2009},
}
@inproceedings{post09ptsg,
  author = {M. Post and D. Gildea},
  booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
  title = {{B}ayesian learning of a tree substitution grammar},
  year = {2009},
}
@inproceedings{gao08comparison,
  author = {J. Gao and M. Johnson},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {344--352},
  title = {A comparison of {B}ayesian estimators for unsupervised Hidden {M}arkov Model {POS} taggers},
  year = {2008},
}
@inproceedings{grenager05segmentation,
  author = {T. Grenager and D. Klein and C. D. Manning},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {371--378},
  title = {Unsupervised learning of field segmentation models for information extraction},
  year = {2005},
}
@article{dasgupta07em,
  author = {S. Dasgupta and L. Schulman},
  journal = {Journal of Machine Learning Research (JMLR)},
  title = {A Probabilistic Analysis of {EM} for Mixtures of Separated, Spherical {G}aussians},
  volume = {8},
  year = {2007},
}
@article{ron98pfa,
  author = {D. Ron and Y. Singer and N. Tishby},
  journal = {Journal of Computer and System Sciences},
  pages = {133--152},
  title = {On the Learnability and Usage of Acyclic Probabilistic Finite Automata},
  volume = {56},
  year = {1998},
}
@inproceedings{srebro06mixture,
  author = {N. Srebro and G. Shakhnarovich and S. Roweis},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {865--872},
  title = {An investigation of computational and informational limits in {G}aussian mixture clustering},
  year = {2006},
}
@inproceedings{feldman05product,
  author = {J. Feldman and R. O'Donnell and R. A. Servedio},
  booktitle = {Foundations of Computer Science (FOCS)},
  pages = {501--510},
  title = {Learning mixtures of product distributions over discrete domains},
  year = {2005},
}
@article{clark05fsa,
  author = {A. Clark and F. Thollard},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {473--497},
  title = {{PAC}-learnability of Probabilistic Deterministic Finite State Automata},
  volume = {5},
  year = {2005},
}
@article{blackwell57identifiable,
  author = {D. Blackwell and L. Koopmans},
  journal = {Annals of Mathematical Statistics},
  pages = {1011--1015},
  title = {On the Identifiability Problem for Functions of Finite {M}arkov Chains},
  volume = {28},
  year = {1957},
}
@techreport{adriaans99shallow,
  author = {P. W. Adriaans},
  institution = {Stanford University},
  title = {Learning Shallow Context-Free Languages under Simple Distributions},
  year = {1999},
}
@article{kuhn55hungarian,
  author = {H. W. Kuhn},
  journal = {Naval Research Logistic Quarterly},
  pages = {83--97},
  title = {The {H}ungarian method for the assignment problem},
  volume = {2},
  year = {1955},
}
@phdthesis{ratnaparkhi98thesis,
  author = {A. Ratnaparkhi},
  school = {University of Pennsylvania},
  title = {Maximum entropy models for natural language ambiguity resolution},
  year = {1998},
}
@inproceedings{mccallum00memm,
  author = {A. McCallum and D. Freitag and F. Pereira},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Maximum Entropy {M}arkov Models for Information Extraction and Segmentation},
  year = {2000},
}
@inproceedings{lafferty01crf,
  author = {J. Lafferty and A. McCallum and F. Pereira},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {282--289},
  title = {Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Data},
  year = {2001},
}
@inproceedings{kakade02objective,
  author = {S. Kakade and Y. W. Teh and S. Roweis},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {An Alternate Objective Function for {M}arkovian Fields},
  year = {2002},
}
@inproceedings{klein02conditional,
  author = {D. Klein and C. D. Manning},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Conditional structure versus conditional estimation in {NLP} models},
  year = {2002},
}
@inproceedings{ng02compare,
  author = {A. Y. Ng and M. I. Jordan},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {On Discriminative vs. Generative classifiers: A comparison of logistic regression and naive {B}ayes},
  year = {2002},
}
@article{white82mis,
  author = {H. White},
  journal = {Econometrica},
  title = {Maximum likelihood estimation of misspecified models},
  volume = {50},
  year = {1982},
}
@article{godambe87quasi,
  author = {V. P. Godambe and C. C. Heyde},
  journal = {International Statistical Review},
  pages = {231--244},
  title = {Quasi-likelihood and Optimal estimation},
  volume = {55},
  year = {1987},
}
@inproceedings{pal06mcl,
  author = {A. McCallum and C. Pal and G. Druck and X. Wang},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Multi-Conditional Learning: Generative/Discriminative Training for Clustering and Classification},
  year = {2006},
}
@inproceedings{lasserre06hybrid,
  author = {J. A. Lasserre and C. M. Bishop and T. P. Minka},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  pages = {87--94},
  title = {Principled Hybrids of Generative and Discriminative Models},
  year = {2006},
}
@inproceedings{raina04hybrid,
  author = {R. Raina and Y. Shen and A. Ng and A. McCallum},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Classification with hybrid generative/discriminative models},
  year = {2004},
}
@inproceedings{suzuki07hybrid,
  author = {J. Suzuki and A. Fujino and H. Isozaki},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  title = {Semi-Supervised Structured Output Learning based on a Hybrid Generative and Discriminative Approach},
  year = {2007},
}
@inproceedings{bouchard04tradeoff,
  author = {G. Bouchard and B. Triggs},
  booktitle = {International Conference on Computational Statistics},
  pages = {721--728},
  title = {The Trade-Off Between Generative and Discriminative Classifiers},
  year = {2004},
}
@book{vaart98asymptotic,
  author = {A. W. van der Vaart},
  publisher = {Cambridge University Press},
  title = {Asymptotic statistics},
  year = {1998},
}
@article{besag75pseudo,
  author = {J. Besag},
  journal = {The Statistician},
  pages = {179--195},
  title = {The analysis of non-lattice data},
  volume = {24},
  year = {1975},
}
@article{lindsay88composite,
  author = {B. Lindsay},
  journal = {Contemporary Mathematics},
  pages = {221--239},
  title = {Composite likelihood methods},
  volume = {80},
  year = {1988},
}
@inproceedings{sutton05piecewise,
  author = {C. Sutton and A. McCallum},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {Piecewise Training of Undirected Models},
  year = {2005},
}
@inproceedings{sutton07pwpl,
  author = {C. Sutton and A. McCallum},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Piecewise Pseudolikelihood for Efficient {CRF} Training},
  year = {2007},
}
@article{cox04pseudo,
  author = {D. R. Cox and N. Reid},
  journal = {Biometrika},
  pages = {729--737},
  title = {A note on pseudolikelihood constructed from marginal densities},
  volume = {91},
  year = {2004},
}
@article{varin08composite,
  author = {C. Varin},
  journal = {Advances in Statistical Analysis},
  pages = {1--28},
  title = {On composite marginal likelihoods},
  volume = {92},
  year = {2008},
}
@article{wainwright06wrong,
  author = {M. Wainwright},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {1829--1859},
  title = {Estimating the ``wrong'' graphical model: Benefits in the computation-limited setting},
  volume = {7},
  year = {2006},
}
@inproceedings{wainwright03trw,
  author = {M. Wainwright and T. Jaakkola and A. Willsky},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {Tree-reweighted belief propagation algorithms and approximate {ML} estimation by pseudo-moment matching},
  year = {2003},
}
@article{hyvarinen06pseudolikelihood,
  author = {A. Hyv\"arinen},
  journal = {Neural Computation},
  pages = {2283--2292},
  title = {Consistency of pseudolikelihood estimation of fully visible {B}oltzmann machines},
  volume = {18},
  year = {2006},
}
@inproceedings{hinton99poe,
  author = {G. Hinton},
  booktitle = {International Conference on Artificial Neural Networks (ICANN)},
  title = {Products of Experts},
  year = {1999},
}
@book{robert04mc,
  author = {C. P. Robert and G. Casella},
  publisher = {Springer},
  title = {{M}onte {C}arlo Statistical Methods},
  year = {2004},
}
@article{wainwright08varinf,
  author = {M. Wainwright and M. I. Jordan},
  journal = {Foundations and Trends in Machine Learning},
  pages = {1--307},
  title = {Graphical models, exponential families, and variational inference},
  volume = {1},
  year = {2008},
}
@article{siepel04phylohmm,
  author = {A. Siepel and D. Haussler},
  journal = {Journal of Computational Biology},
  pages = {413--428},
  title = {Combining phylogenetic and hidden {M}arkov models in biosequence analysis},
  volume = {11},
  year = {2004},
}
@article{felsenstein96phylohmm,
  author = {J. Felsenstein and G. A. Churchill},
  journal = {Molecular Biology and Evolution},
  pages = {93--104},
  title = {A hidden {M}arkov model approach to variation among sites in rate of evolution},
  volume = {13},
  year = {1996},
}
@article{jojic04phylohmm,
  author = {V. Jojic and N. Jojic and C. Meek and D. Geiger and A. Siepel and D. Haussler and D. Heckerman},
  journal = {Bioinformatics},
  pages = {161--168},
  title = {Efficient approximations for learning phylogenetic {HMM} models from data},
  volume = {20},
  year = {2004},
}
@inproceedings{geiger05structured,
  author = {D. Geiger and C. Meek},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {Structured Variational Inference Procedures and their Realizations},
  year = {2005},
}
@inproceedings{shotton06textonboost,
  author = {J. Shotton and J. Winn and C. Rother and A. Criminisi},
  booktitle = {European Conference on Computer Vision (ECCV)},
  title = {TextonBoost: Joint Appearance, Shape and Context Modeling for Multi-Class Object Recognition and Segmentation},
  year = {2006},
}
@article{zhu06grammar,
  author = {S. C. Zhu and D. Mumford},
  journal = {Foundations and Trends in Computer Graphics and Vision},
  pages = {259--362},
  title = {A Stochastic Grammar of Images},
  volume = {2},
  year = {2006},
}
@article{sakakibara05grammar,
  author = {Y. Sakakibara},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {1051--1062},
  title = {Grammatical Inference in Bioinformatics},
  volume = {27},
  year = {2005},
}
@inproceedings{denero08phrase,
  author = {J. DeNero and A. Bouchard-C\^ot\'e and D. Klein},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {314--323},
  title = {Sampling Alignment Structure under a {B}ayesian Translation Model},
  year = {2008},
}
@inproceedings{wallach08dependency,
  author = {H. Wallach and C. Sutton and A. McCallum},
  booktitle = {Workshop on Prior Knowledge for Text and Language},
  pages = {15--20},
  title = {{B}ayesian Modeling of Dependency Trees Using Hierarchical {P}itman-{Y}or Priors},
  year = {2008},
}
@inproceedings{blunsom09synchronous,
  author = {P. Blunsom and T. Cohn and M. Osborne},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {{B}ayesian Synchronous Grammar Induction},
  year = {2009},
}
@article{wu97itg,
  author = {D. Wu},
  journal = {Computational Linguistics},
  pages = {377--404},
  title = {Stochastic inversion transduction grammars and bilingual parsing of parallel corpora},
  volume = {23},
  year = {1997},
}
@article{dyrka07pcfg,
  author = {W. Dyrka and J. Nebel},
  journal = {Systems Biology, Bioinformatics and Synthetic Biology},
  pages = {78--79},
  title = {A probabilistic context-free grammar for the detection of binding sites from a protein sequence},
  volume = {1},
  year = {2007},
}
@inproceedings{saxena07scene,
  author = {A. Saxena and M. Sun and A. Y. Ng},
  booktitle = {International Conference on Computer Vision (ICCV)},
  title = {Learning 3-{D} Scene Structure from a Single Still Image},
  year = {2007},
}
@inproceedings{delage06dbn,
  author = {E. Delage and H. Lee and A. Y. Ng},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  title = {A dynamic {B}ayesian network model for autonomous 3d reconstruction},
  year = {2006},
}
@inproceedings{hoiem05geometric,
  author = {D. Hoiem and A. A. Efros and M. Herbert},
  booktitle = {International Conference on Computer Vision (ICCV)},
  title = {Geometric Context from a Single Image},
  year = {2005},
}
@inproceedings{hoiem07occlusion,
  author = {D. Hoiem and A. N. Stein and A. A. Efros and M. Herbert},
  booktitle = {International Conference on Computer Vision (ICCV)},
  title = {Recovering Occlusion Boundaries from a Single Image},
  year = {2007},
}
@inproceedings{leung96repeat,
  author = {T. Leung and J. Malik},
  booktitle = {European Conference on Computer Vision (ECCV)},
  title = {Detecting, Localizing and Grouping Repeated Scene Elements from an Image},
  year = {1996},
}
@inproceedings{ren06figure,
  author = {X. Ren and C. C. Fowlkes and J. Malik},
  booktitle = {European Conference on Computer Vision (ECCV)},
  title = {Figure/Ground Assignment in Natural Images},
  year = {2006},
}
@book{pollard84convergence,
  author = {D. Pollard},
  publisher = {Springer-Verlag},
  title = {Convergence of Stochastic Processes},
  year = {1984},
}
@article{zhang02covering,
  author = {T. Zhang},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {527--550},
  title = {Covering number bounds of certain regularized linear function classes},
  volume = {2},
  year = {2002},
}
@inproceedings{collins01theory,
  author = {M. Collins},
  booktitle = {International Workshop on Parsing Technologies},
  title = {Parameter Estimation for Statistical Parsing Models: Theory and Practice of Distribution-Free Methods},
  year = {2001},
}
@article{csiszar04info,
  author = {I. Csisz\'ar and P. Shields},
  journal = {Foundations and Trends in Communications and Information Theory},
  pages = {417--528},
  title = {Information Theory and Statistics: A Tutorial},
  volume = {1},
  year = {2004},
}
@inproceedings{bartlett01rademacher,
  author = {P. L. Bartlett and S. Mendelson},
  booktitle = {Conference on Learning Theory (COLT)},
  pages = {224--240},
  title = {{R}ademacher and {G}aussian complexities: Risk bounds and structural results},
  year = {2001},
}
@book{shawetaylor04kernel,
  author = {J. Shawe-Taylor and N. Cristianini},
  publisher = {Cambridge University Press},
  title = {Kernel Methods for Pattern Analysis},
  year = {2004},
}
@inproceedings{crammar07multiple,
  author = {K. Crammar and M. Kearns and J. Wortman},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Learning from Multiple Sources},
  year = {2007},
}
@inproceedings{punyakanok05constrained,
  author = {V. Punyakanok and D. Roth and W. Yih and D. Zimak},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  title = {Learning and Inference over Constrained Output},
  year = {2005},
}
@inproceedings{bucila06compress,
  author = {C. Bucil\u{a} and R. Caruana and A. Niculescu-Mizil},
  booktitle = {International Conference on Knowledge Discovery and Data Mining (KDD)},
  title = {Model Compression},
  year = {2006},
}
@phdthesis{craven96trepan,
  author = {M. W. Craven},
  school = {University of Wisconsin at Madison},
  title = {Extracting comprehensible models from trained neural networks},
  year = {1996},
}
@book{krishnamurthi06pl,
  author = {S. Krishnamurthi},
  publisher = {Creative Commons},
  title = {Programming Languages: Application and Interpretation},
  year = {2006},
}
@article{wand87simple,
  author = {M. Wand},
  journal = {Fundamenta Informaticae},
  pages = {115--122},
  title = {A Simple Algorithm and Proof for Type Inference},
  volume = {10},
  year = {1987},
}
@inproceedings{kothari08polylet,
  author = {S. Kothari and J. L. Caldwell},
  booktitle = {Fourth Conference on Computability},
  title = {On Extending {W}and's Type Reconstruction Algorithm to Handle Polymorphic Let},
  year = {2008},
}
@techreport{kothari07survey,
  author = {S. Kothari},
  institution = {University of Wyoming},
  title = {Type Reconstruction Algorithms - A Survey},
  year = {2007},
}
@techreport{pottier05modern,
  author = {F. Pottier},
  institution = {INRIA},
  title = {A Modern Eye on {ML} Type Inference: Old Techniques and Recent Developments},
  year = {2005},
}
@article{collins08exponentiated,
  author = {M. Collins and A. Globerson and T. Koo and X. Carreras and P. Bartlett},
  journal = {Journal of Machine Learning Research (JMLR)},
  title = {Exponentiated Gradient Algorithms for Conditional Random Fields and Max-Margin {M}arkov Networks},
  volume = {9},
  year = {2008},
}
@inproceedings{finkel08crf,
  author = {J. R. Finkel and A. Kleeman and C. Manning},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  title = {Efficient, Feature-based, Conditional Random Field Parsing},
  year = {2008},
}
@inproceedings{neal98incremental,
  author = {R. Neal and G. Hinton},
  booktitle = {Learning in Graphical Models},
  title = {A view of the {EM} algorithm that justifies incremental, sparse, and other variants},
  year = {1998},
}
@article{cappe09online,
  author = {O. Capp\'e and E. Moulines},
  journal = {Journal of the Royal Statistics Society: Series B (Statistical Methodology)},
  pages = {593--613},
  title = {Online Expectation-Maximization Algorithm for Latent Data Models},
  volume = {71},
  year = {2009},
}
@inproceedings{collins02perceptron,
  author = {M. Collins},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Discriminative Training Methods for Hidden {M}arkov Models: Theory and Experiments with {P}erceptron Algorithms},
  year = {2002},
}
@article{och03systematic,
  author = {F. J. Och and H. Ney},
  journal = {Computational Linguistics},
  pages = {19--51},
  title = {A Systematic Comparison of Various Statistical Alignment Models},
  volume = {29},
  year = {2003},
}
@article{sato00online,
  author = {M. Sato and S. Ishii},
  journal = {Neural Computation},
  pages = {407--432},
  title = {On-line {EM} Algorithm for the Normalized {G}aussian Network},
  volume = {12},
  year = {2000},
}
@inproceedings{mcdonald05online,
  author = {R. McDonald and K. Crammer and F. Pereira},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {91--98},
  title = {Online Large-Margin Training of Dependency Parsers},
  year = {2005},
}
@inproceedings{bottou08large,
  author = {L. Bottou and O. Bousquet},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {The Tradeoffs of Large Scale Learning},
  year = {2008},
}
@inproceedings{shalev08inverse,
  author = {S. Shalev-Shwartz and N. Srebro},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {{SVM} Optimization: Inverse Dependence on Training Set Size},
  year = {2008},
}
@inproceedings{seginer07fast,
  author = {Y. Seginer},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Fast Unsupervised Incremental Parsing},
  year = {2007},
}
@inproceedings{yarowsky95unsupervised,
  author = {D. Yarowsky},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Unsupervised word sense disambiguation rivaling supervised methods},
  year = {1995},
}
@inproceedings{salakhutdinov03ecg,
  author = {R. Salakhutdinov and S. Roweis and Z. Ghahramani},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Optimization with {EM} and expectation-conjugate-gradient},
  year = {2003},
}
@inproceedings{gildea99topic,
  author = {D. Gildea and T. Hofmann},
  booktitle = {Eurospeech},
  title = {Topic-based language models using {EM}},
  year = {1999},
}
@inproceedings{kuo08transliterations,
  author = {J. Kuo and H. Li and C. Lin},
  booktitle = {Sixth SIGHAN Workshop on Chinese Language Processing},
  title = {Mining Transliterations from Web Query Results: An Incremental Approach},
  year = {2008},
}
@article{venkataraman01word,
  author = {A. Venkataraman},
  journal = {Computational Linguistics},
  pages = {351--372},
  title = {A statistical model for word discovery in transcribed speech},
  volume = {27},
  year = {2001},
}
@inproceedings{quadrianto08labels,
  author = {N. Quadrianto and A. J. Smola and T. S. Caetano and Q. V. Le},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {776--783},
  title = {Estimating labels from label proportions},
  year = {2008},
}
@inproceedings{ganchev08multi,
  author = {K. Ganchev and J. Graça and J. Blitzer and B. Taskar},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {Multi-View Learning over Structured and Non-Identical Outputs},
  year = {2008},
}
@inproceedings{graca08em,
  author = {J. Graça and K. Ganchev and B. Taskar},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {569--576},
  title = {Expectation Maximization and Posterior Constraints},
  year = {2008},
}
@inproceedings{druck08ge,
  author = {G. Druck and G. Mann and A. McCallum},
  booktitle = {ACM Special Interest Group on Information Retreival (SIGIR)},
  pages = {595--602},
  title = {Learning from Labeled Features using Generalized Expectation Criteria},
  year = {2008},
}
@inproceedings{mann07expectation,
  author = {G. Mann and A. McCallum},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {593--600},
  title = {Simple, Robust, Scalable Semi-supervised Learning via Expectation Regularization},
  year = {2007},
}
@inproceedings{mann08ge,
  author = {G. Mann and A. McCallum},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  pages = {870--878},
  title = {Generalized Expectation Criteria for Semi-Supervised Learning of Conditional Random Fields},
  year = {2008},
}
@inproceedings{chang07constraint,
  author = {M. Chang and L. Ratinov and D. Roth},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {280--287},
  title = {Guiding Semi-Supervision with Constraint-Driven Learning},
  year = {2007},
}
@inproceedings{haghighi06prototype,
  author = {A. Haghighi and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {320--327},
  title = {Prototype-Driven Learning for Sequence Models},
  year = {2006},
}
@inproceedings{zaidan08annotator,
  author = {O. F. Zaidan and J. Eisner},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Modeling Annotators: A Generative Approach to Learning from Annotator Rationales},
  year = {2008},
}
@inproceedings{zaidan07annotator,
  author = {O. F. Zaidan and J. Eisner and C. D. Piatko},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  title = {Using "Annotator Rationales" to Improve Machine Learning for Text Categorization},
  year = {2007},
}
@inproceedings{moore04improving,
  author = {R. C. Moore},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {518--525},
  title = {Improving {IBM} Word Alignment Model 1},
  year = {2004},
}
@inproceedings{daume04alignment,
  author = {H. Daume and D. Marcu},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Phrase-Based {HMM} Approach to Document/Abstract Alignment},
  year = {2004},
}
@inproceedings{barzilay05content,
  author = {R. Barzilay and M. Lapata},
  booktitle = {Human Language Technology and Empirical Methods in Natural Language Processing (HLT/EMNLP)},
  pages = {331--338},
  title = {Collective Content Selection for Concept-To-Text Generation},
  year = {2005},
}
@inproceedings{snyder07database,
  author = {B. Snyder and R. Barzilay},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  pages = {1713--1718},
  title = {Database-Text Alignment via Structured Multilabel Classification},
  year = {2007},
}
@inproceedings{zettlemoyer05ccg,
  author = {L. S. Zettlemoyer and M. Collins},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  pages = {658--666},
  title = {Learning to Map Sentences to Logical Form: Structured Classification with Probabilistic Categorial Grammars},
  year = {2005},
}
@inproceedings{zettlemoyer07relaxed,
  author = {L. S. Zettlemoyer and M. Collins},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {678--687},
  title = {Online Learning of Relaxed {CCG} Grammars for Parsing to Logical Form},
  year = {2007},
}
@inproceedings{lu08generative,
  author = {W. Lu and H. T. Ng and W. S. Lee and L. S. Zettlemoyer},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {783--792},
  title = {A Generative Model for Parsing Natural Language to Meaning Representations},
  year = {2008},
}
@inproceedings{ge05scissor,
  author = {R. Ge and R. J. Mooney},
  booktitle = {Computational Natural Language Learning (CoNLL)},
  pages = {9--16},
  title = {A Statistical Semantic Parser that Integrates Syntax and Semantics},
  year = {2005},
}
@inproceedings{kate06krisp,
  author = {R. J. Kate and R. J. Mooney},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  pages = {913--920},
  title = {Using String-Kernels for Learning Semantic Parsers},
  year = {2006},
}
@inproceedings{kate07krisper,
  author = {R. J. Kate and R. J. Mooney},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {895--900},
  title = {Learning Language Semantics from Ambiguous Supervision},
  year = {2007},
}
@inproceedings{wong06mt,
  author = {Y. W. Wong and R. J. Mooney},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {439--446},
  title = {Learning for semantic parsing with statistical machine translation},
  year = {2006},
}
@inproceedings{wong07synchronous,
  author = {Y. W. Wong and R. J. Mooney},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {960--967},
  title = {Learning Synchronous Grammars for Semantic Parsing with Lambda Calculus},
  year = {2007},
}
@inproceedings{chen08sportscast,
  author = {D. L. Chen and R. J. Mooney},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {128--135},
  title = {Learning to Sportscast: A Test of Grounded Language Acquisition},
  year = {2008},
}
@article{feldman04embodied,
  author = {J. Feldman and S. Narayanan},
  journal = {Brain and Language},
  pages = {385--392},
  title = {Embodied Meaning in a Neural Theory of Language},
  volume = {89},
  year = {2004},
}
@inproceedings{branavan08annotation,
  author = {S. Branavan and H. Chen and J. Eisenstein and R. Barzilay},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Learning Document-Level Semantic Properties from Free-text Annotations},
  year = {2008},
}
@article{barzilay08coherence,
  author = {R. Barzilay and M. Lapata},
  journal = {Computational Linguistics},
  pages = {1--34},
  title = {Modeling Local Coherence: An Entity-based Approach},
  volume = {34},
  year = {2008},
}
@inproceedings{eisenstein08topic,
  author = {J. Eisenstein and R. Barzilay},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {334--343},
  title = {{B}ayesian Unsupervised Topic Segmentation},
  year = {2008},
}
@inproceedings{barzilay06aggregation,
  author = {R. Barzilay and M. Lapata},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  title = {Aggregation via Set Partitioning for Natural Language Generation},
  year = {2006},
}
@inproceedings{barzilay04content,
  author = {R. Barzilay and L. Lee},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  title = {Catching the Drift: Probabilistic Content Models, with Applications to Generation and Summarization},
  year = {2004},
}
@article{siskind96cross,
  author = {J. M. Siskind},
  journal = {Cognition},
  pages = {1--38},
  title = {A computational study of cross-situational techniques for learning word-to-meaning mappings},
  volume = {61},
  year = {1996},
}
@article{gorniak07situated,
  author = {P. Gorniak and D. Roy},
  journal = {Cognitive Science},
  pages = {197--231},
  title = {Situated Language Understanding as Filtering Perceived Affordances},
  volume = {31},
  year = {2007},
}
@inproceedings{yu04grounding,
  author = {C. Yu and D. H. Ballard},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {488--493},
  title = {On the integration of grounding language and learning objects},
  year = {2004},
}
@inproceedings{grenager06verb,
  author = {T. Grenager and C. D. Manning},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Unsupervised discovery of a statistical verb lexicon},
  year = {2006},
}
@inproceedings{poon09semantic,
  author = {H. Poon and P. Domingos},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Unsupervised Semantic Parsing},
  year = {2009},
}
@inproceedings{zettlemoyer09context,
  author = {L. S. Zettlemoyer and M. Collins},
  booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
  title = {Learning Context-dependent Mappings from Sentences to Logical Form},
  year = {2009},
}
@inproceedings{branavan09reinforcement,
  author = {S. Branavan and H. Chen and L. S. Zettlemoyer and R. Barzilay},
  booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
  pages = {82--90},
  title = {Reinforcement Learning for Mapping Instructions to Actions},
  year = {2009},
}
@article{yu07unified,
  author = {C. Yu and D. H. Ballard},
  journal = {Neurocomputing},
  number = {13},
  pages = {2149--2165},
  title = {A Unified Model of Early Word Learning: Integrating Statistical and Social Cues},
  volume = {70},
  year = {2007},
}
@article{frank09intentions,
  author = {M. C. Frank and N. D. Goodman and J. B. Tenenbaum},
  journal = {Psychological Science},
  number = {5},
  pages = {578--585},
  title = {Using Speakers' Referential Intentions to Model Early Cross-Situational Word Learning},
  volume = {20},
  year = {2009},
}
@article{chaloner95bayesian,
  author = {K. Chaloner and I. Verdinelli},
  journal = {Statistical Science},
  pages = {273--304},
  title = {{B}ayesian Experimental Design: A Review},
  volume = {10},
  year = {1995},
}
@inproceedings{seeger08cs,
  author = {M. Seeger and H. Nickisch},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {912--919},
  title = {Compressed Sensing and {B}ayesian Experimental Design},
  year = {2008},
}
@article{nemirovski02saddle,
  author = {A. Nemirovski and R. Y. Rubinstein},
  journal = {International Series in Operations Research and Management Science},
  pages = {155--184},
  title = {An Efficient Stochastic Approximation Algorithm for Stochastic Saddle Point Problems},
  volume = {46},
  year = {2002},
}
@phdthesis{schein05active,
  author = {A. I. Schein},
  school = {Carnegie Mellon University},
  title = {Active Learning for Logistic Regression},
  year = {2005},
}
@inproceedings{jaakkola97logistic,
  author = {T. Jaakkola and M. I. Jordan},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {A variational approach to {B}ayesian logistic regression models and their extensions},
  year = {1997},
}
@article{dudik07maxent,
  author = {M. Dudík and S. J. Phillips and R. E. Schapire},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {1217--1260},
  title = {Maximum Entropy Density Estimation},
  volume = {8},
  year = {2007},
}
@book{borwein05variational,
  author = {J. M. Borwein and Q. J. Zhu},
  publisher = {Springer},
  title = {Techniques of Variational Analysis},
  year = {2005},
}
@inproceedings{roy01active,
  author = {N. Roy and A. McCallum},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {441--448},
  title = {Toward Optimal Active Learning through Sampling Estimation of Error Reduction},
  year = {2001},
}
@article{brown92class,
  author = {P. F. Brown and V. J. D. Pietra and P. V. deSouza and J. C. Lai and R. L. Mercer},
  journal = {Computational Linguistics},
  pages = {467--479},
  title = {Class-Based n-gram Models of Natural Language},
  volume = {18},
  year = {1992},
}
@inproceedings{grandvalet05entropy,
  author = {Y. Grandvalet and Y. Bengio},
  booktitle = {Semi-Supervised Learning},
  title = {Entropy Regularization},
  year = {2005},
}
@inproceedings{stein61quadratic,
  author = {W. James and C. Stein},
  booktitle = {Fourth Berkeley Symposium in Mathematics, Statistics, and Probability},
  pages = {361--380},
  title = {Estimation with Quadratic Loss},
  year = {1961},
}
@inproceedings{bouchard07tradeoff,
  author = {G. Bouchard},
  booktitle = {Sixth International Conference on Machine Learning and Applications (ICMLA)},
  pages = {124--129},
  title = {Bias-Variance Tradeoff in Hybrid Generative-Discriminative Models},
  year = {2007},
}
@article{akaike74aic,
  author = {H. Akaike},
  journal = {IEEE Transactions on Automatic Control},
  pages = {716--723},
  title = {A new look at the statistical model identification},
  volume = {19},
  year = {1974},
}
@article{bakker03task,
  author = {B. Bakker and T. Heskes},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {83--99},
  title = {Task clustering and gating for {B}ayesian multitask learning},
  volume = {4},
  year = {2003},
}
@article{evgeniou05task,
  author = {T. Evgeniou and C. Micchelli and M. Pontil},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {615--637},
  title = {Learning multiple tasks with kernel methods},
  volume = {6},
  year = {2005},
}
@inproceedings{argyriou07feature,
  author = {A. Argyriou and T. Evgeniou and M. Pontil},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {41--48},
  title = {Multi-task feature learning},
  year = {2007},
}
@inproceedings{jacob09cluster,
  author = {L. Jacob and F. Bach and J. Vert},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {745--752},
  title = {Clustered Multi-Task Learning: A Convex Formulation},
  year = {2009},
}
@article{bickel06regularization,
  author = {P. Bickel and B. Li},
  journal = {Sociedad de Estadística e Investigación Operativa Test},
  pages = {271--344},
  title = {Regularization in Statistics},
  volume = {15},
  year = {2006},
}
@article{bernardo79reference,
  author = {J. M. Bernardo},
  journal = {Journal of the Royal Statistics Society: Series B (Statistical Methodology)},
  pages = {113--147},
  title = {Reference posterior distributions for {B}ayesian inference},
  volume = {41},
  year = {1979},
}
@article{peters06bindingprediction,
  author = {B. Peters and H. Bui and S. Frankild and M. Nielson and C. Lundegaard and E. Kostem and D. Basch and K. Lamberth and M. Harndahl and W. Fleri and S. S. Wilson and J. Sidney and O. Lund and S. Buus and A. Sette},
  journal = {PLoS Compututational Biology},
  title = {A community resource benchmarking predictions of peptide binding to {MHC-I} molecules},
  volume = {2},
  year = {2006},
}
@article{craven78gcv,
  author = {P. Craven and G. Wahba},
  journal = {Numerische Mathematik},
  number = {4},
  pages = {377--403},
  title = {Smoothing noisy data with spline functions. Estimating the correct degree of smoothing by the method of generalized cross-validation},
  volume = {31},
  year = {1978},
}
@article{eldar09sure,
  author = {Y. C. Eldar},
  journal = {IEEE Transactions on Signal Processing},
  number = {2},
  pages = {471--481},
  title = {Generalized {SURE} for Exponential Families: Applications to Regularization},
  volume = {57},
  year = {2009},
}
@article{stein81sure,
  author = {C. M. Stein},
  journal = {Annals of Statistics},
  number = {6},
  pages = {1135--1151},
  title = {Estimation of the mean of a multivariate normal distribution},
  volume = {9},
  year = {1981},
}
@article{bartlett05local,
  author = {P. L. Bartlett and O. Bousquet and S. Mendelson},
  journal = {Annals of Statistics},
  number = {4},
  pages = {1497--1537},
  title = {Local {R}ademacher complexities},
  volume = {33},
  year = {2005},
}
@article{bousquet02stability,
  author = {O. Bousquet and A. Elisseeff},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {499--526},
  title = {Stability and Generalization},
  volume = {2},
  year = {2002},
}
@book{cesabianchi06prediction,
  author = {N. Cesa-Bianchi and G. Lugosi},
  publisher = {Cambridge University Press},
  title = {Prediction, learning, and games},
  year = {2006},
}
@article{bartlett53approximate,
  author = {M. S. Bartlett},
  journal = {Biometrika},
  pages = {306--317},
  title = {Approximate confidence intervals. {II}. {M}ore than one unknown parameter},
  volume = {40},
  year = {1953},
}
@article{levit85second,
  author = {B. Y. Levit},
  journal = {Theory of Probability and its Applications},
  pages = {333--363},
  title = {Second-order asymptotic optimality and positive solutions of the Schrödinger equation},
  volume = {30},
  year = {1985},
}
@article{berkhin80second,
  author = {P. E. Berkhin and B. Y. Levit},
  journal = {Problemy Peredachi Informatsii},
  pages = {60--79},
  title = {Second-order asymptotically minimax estimates for the mean of a normal population},
  volume = {16},
  year = {1980},
}
@article{dalalyan06second,
  author = {A. S. Dalalyan and G. K. Golubev and A. B. Tsybakov},
  journal = {Annals of Statistics},
  number = {1},
  pages = {169--201},
  title = {Penalized maximum likelihood and semiparametric second-order efficiency},
  volume = {34},
  year = {2006},
}
@incollection{shibata89model,
  author = {R. Shibata},
  booktitle = {From Data to Model},
  pages = {215--240},
  title = {Statistical Aspects of Model Selection},
  year = {1989},
}
@article{murata94neural,
  author = {N. Murata and S. Yoshizawa and S. Amari},
  journal = {IEEE Transactions on Neural Networks},
  number = {6},
  pages = {865--872},
  title = {Network Information Criterion---Determining the Number of Hidden Units for an Artificial Neural Network Model},
  volume = {5},
  year = {1994},
}
@article{konishi96gic,
  author = {S. Konishi and G. Kitagawa},
  journal = {Biometrika},
  number = {4},
  pages = {875--890},
  title = {Generalized Information Criteria in Model Selection},
  volume = {83},
  year = {1996},
}
@article{ishiguro97bootstrapping,
  author = {M. Ishiguro and Y. Sakamoto and G. Kitagawa},
  journal = {Annals of the Institute of Statistical Mathematics},
  pages = {411--434},
  title = {Bootstrapping Log Likelihood and {EIC}, an Extension of {AIC}},
  volume = {49},
  year = {1997},
}
@inproceedings{sahavechaphan06xsnippet,
  author = {N. Sahavechaphan and K. Claypool},
  booktitle = {Object-Oriented Programming, Systems, Languages, and Applications (OOPSLA)},
  title = {XSnippet: Mining For Sample Code},
  volume = {41},
  year = {2006},
}
@inproceedings{jiang07deckard,
  author = {L. Jiang and G. Misherghi and Z. Su and S. Glondu},
  booktitle = {International Conference on Software Engineering (ICSE)},
  pages = {96--105},
  title = {{DECKARD}: Scalable and Accurate Tree-Based Detection of Code Clones},
  year = {2007},
}
@inproceedings{lu09generation,
  author = {W. Lu and H. T. Ng and W. S. Lee},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {400--409},
  title = {Natural Language Generation with Tree Conditional Random Fields},
  year = {2009},
}
@article{ratnaparkhi02surface,
  author = {A. Ratnaparkhi},
  journal = {Computer, Speech \& Language},
  pages = {435--455},
  title = {Trainable approaches to surface natural language generation and their application to conversational dialog systems},
  volume = {16},
  year = {2002},
}
@inproceedings{mellish00dependencies,
  author = {C. Mellish and R. Evans and L. Cahill and C. Doran and D. Paiva and M. Reape and D. Scott and N. Tipper},
  booktitle = {Applied Natural Language Processing and North American Association for Computational Linguistics (ANLP/NAACL)},
  pages = {119--126},
  title = {A Representation for Complex and Evolving Data Dependencies in Generation},
  year = {2000},
}
@inproceedings{koller02generation,
  author = {A. Koller and K. Striegnitz},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {17--24},
  title = {Generation as Dependency Parsing},
  year = {2002},
}
@inproceedings{paiva05control,
  author = {D. S. Paiva and R. Evans},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {58--65},
  title = {Empirically-based Control of Natural Language Generation},
  year = {2005},
}
@inproceedings{soricut06widl,
  author = {R. Soricut and D. Marcu},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {1105--1112},
  title = {Stochastic Language Generation Using {WIDL}-Expressions and its Application in Machine Translation and Summarization},
  year = {2006},
}
@inproceedings{soricut06discourse,
  author = {R. Soricut and D. Marcu},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {803--810},
  title = {Discourse Generation Using Utility-Trained Coherence Models},
  year = {2006},
}
@inproceedings{denero09efficient,
  author = {J. DeNero and M. Bansal and A. Pauls and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {227--235},
  title = {Efficient Parsing for Transducer Grammars},
  year = {2009},
}
@inproceedings{agarwal09hybrid,
  author = {A. Agarwal and H. Daume},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  title = {Exponential Family Hybrid Learning},
  year = {2009},
}
@article{belz08sumtime,
  author = {A. Belz},
  journal = {Natural Language Engineering},
  number = {4},
  pages = {1--26},
  title = {Automatic generation of weather forecast texts using comprehensive probabilistic generation-space models},
  volume = {14},
  year = {2008},
}
@article{reiter05sumtime,
  author = {E. Reiter and S. Sripada and J. Hunter and J. Yu and I. Davy},
  journal = {Artificial Intelligence},
  pages = {137--169},
  title = {Choosing words in computer-generated weather forecasts},
  volume = {167},
  year = {2005},
}
@inproceedings{belz09sumtime,
  author = {A. Belz and E. Kow},
  booktitle = {European Workshop on Natural Language Generation},
  pages = {16--24},
  title = {System building cost vs. output quality in data-to-text generation},
  year = {2009},
}
@inproceedings{green06biomedical,
  author = {N. Green},
  booktitle = {International Natural Language Generation Conference},
  pages = {114--121},
  title = {Generation of biomedical arguments for lay readers},
  year = {2006},
}
@inproceedings{turner09geo,
  author = {R. Turner and Y. Sripada and E. Reiter},
  booktitle = {European Workshop on Natural Language Generation},
  pages = {42--49},
  title = {Generating approximate geographic descriptions},
  year = {2009},
}
@inproceedings{dale03navigation,
  author = {R. Dale and S. Geldof and J. Prost},
  booktitle = {Australasian computer science conference},
  pages = {35--44},
  title = {CORAL: using natural language generation for navigational assistance},
  year = {2003},
}
@inproceedings{foster04xslt,
  author = {M. E. Foster and M. White},
  booktitle = {Workshop on NLP and XML: RDF/RDFS and OWL in Language Technology},
  pages = {1--8},
  title = {Techniques for text planning with {XSLT}},
  year = {2004},
}
@article{liu89lbfgs,
  author = {D. C. Liu and J. Nocedal},
  journal = {Mathematical Programming B},
  number = {3},
  pages = {503--528},
  title = {On the Limited Memory Method for Large Scale Optimization},
  volume = {45},
  year = {1989},
}
@inproceedings{papineni02bleu,
  author = {K. Papineni and S. Roukos and T. Ward and W. Zhu},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {{BLEU}: A Method for Automatic Evaluation of Machine Translation},
  year = {2002},
}
@inproceedings{piantadosi08compositional,
  author = {S. T. Piantadosi and N. D. Goodman and B. A. Ellis and J. B. Tenenbaum},
  booktitle = {Proceedings of the Thirtieth Annual Conference of the Cognitive Science Society},
  pages = {1620--1625},
  title = {A {B}ayesian Model of the Acquisition of Compositional Semantics},
  year = {2008},
}
@inproceedings{goodman08church,
  author = {N. D. Goodman and V. K. Mansighka and D. Roy and K. Bonawitz and J. B. Tenenbaum},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {Church: a language for generative models},
  year = {2008},
}
@article{lau03programming,
  author = {T. Lau and S. Wolfman and P. Domingos and D. S. Weld},
  journal = {Machine Learning},
  pages = {111--156},
  title = {Programming by demonstration using version space algebra},
  volume = {53},
  year = {2003},
}
@inproceedings{lau03traces,
  author = {T. Lau and P. Domingos and D. S. Weld},
  booktitle = {International Conference On Knowledge Capture},
  pages = {36--43},
  title = {Learning Programs from Traces using Version Space Algebra},
  year = {2003},
}
@inproceedings{briggs06functional,
  author = {F. Briggs and M. O'Neill},
  booktitle = {Third Asian-Pacific workshop on Genetic Programming},
  pages = {110--127},
  title = {Functional genetic programming with combinators},
  year = {2006},
}
@inproceedings{blunsom09note,
  author = {P. Blunsom and T. Cohn and S. Goldwater and M. Johnson},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {A note on the implementation of Hierarchical {D}irichlet Processes},
  year = {2009},
}
@book{cypher93pbd,
  author = {A. Cypher},
  publisher = {MIT Press},
  title = {Watch what {I} do: Programming by demonstration},
  year = {1993},
}
@article{caruana97multitask,
  author = {R. Caruana},
  journal = {Machine Learning},
  pages = {41--75},
  title = {Multitask learning},
  volume = {28},
  year = {1997},
}
@article{xue07multitask,
  author = {Y. Xue and X. Liao and L. Carin and B. Krishnapuram},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {35--63},
  title = {Multi-task learning for classification with {D}irichlet process priors},
  volume = {8},
  year = {2007},
}
@book{hankin04lambda,
  author = {C. Hankin},
  publisher = {Lightning Source},
  title = {An Introduction to Lambda Calculi for Computer Scientists},
  year = {2004},
}
@inproceedings{ariola97cyclic,
  author = {Z. M. Ariola and S. Blom},
  booktitle = {Theoretical Aspects of Computer Software},
  pages = {77--106},
  title = {Cyclic lambda calculi},
  year = {1997},
}
@book{vapnik00nature,
  author = {V. N. Vapnik},
  publisher = {Springer-Verlag},
  title = {The Nature of Statistical Learning Theory},
  year = {2000},
}
@inproceedings{haussler94rigorous,
  author = {D. Haussler and M. Kearns and H. S. Seung and N. Tishby},
  booktitle = {Conference on Learning Theory (COLT)},
  pages = {76--87},
  title = {Rigorous Learning Curve Bounds from Statistical Mechanics},
  year = {1994},
}
@article{wanwright09sharp,
  author = {M. J. Wainwright},
  journal = {IEEE Transactions on Information Theory},
  pages = {2183--2202},
  title = {Sharp thresholds for noisy and high-dimensional recovery of sparsity using $\ell_1$-constrained quadratic programming (Lasso)},
  volume = {55},
  year = {2009},
}
@article{schoenfinkel24combinatory,
  author = {M. Schönfinkel},
  journal = {Mathematische Annalen},
  pages = {305--316},
  title = {Über die Bausteine der mathematischen Logik},
  volume = {92},
  year = {1924},
}
@phdthesis{shalev07online,
  author = {S. Shalev-Shwartz},
  school = {The Hebrew University of Jerusalem},
  title = {Online Learning: Theory, Algorithms, and Applications},
  year = {2007},
}
@inproceedings{eisenstein09read,
  author = {J. Eisenstein and J. Clarke and D. Goldwasser and D. Roth},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {958--967},
  title = {Reading to Learn: Constructing Features from Semantic Abstracts},
  year = {2009},
}
@article{culy96null,
  author = {C. Culy},
  journal = {Language Variation and Change},
  pages = {91--124},
  title = {Null objects in {E}nglish recipes},
  volume = {8},
  year = {1996},
}
@inproceedings{fleischman07intention,
  author = {M. Fleischman and D. Roy},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Representing Intentions in a Cognitive Model of Language Acquisition: Effects of Phrase Structure on Situated Verb Learning},
  year = {2007},
}
@inproceedings{solar05sketching,
  author = {A. Solar-Lezama and R. Rabbah and R. Bodík and K. Ebcioglu},
  booktitle = {Programming Language Design and Implementation (PLDI)},
  title = {Programming by Sketching for Bit-Streaming Programs},
  year = {2005},
}
@techreport{srebro10stochastic,
  author = {N. Srebro and K. Sridharan and A. Tewari},
  institution = {TTI Chicago},
  title = {Stochastic optimization and online learning with smooth loss functions},
  year = {2010},
}
@phdthesis{mansinghka09thesis,
  author = {V. Mansinghka},
  school = {MIT},
  title = {Natively Probabilistic Computation},
  year = {2009},
}
@article{goodman08rule,
  author = {N. D. Goodman and J. B. Tenenbaum and J. Feldman and T. L. Griffiths},
  journal = {Cognitive Science},
  pages = {108--154},
  title = {A rational analysis of rule-based concept learning},
  volume = {32},
  year = {2008},
}
@inproceedings{arlot10penalty,
  author = {S. Arlot and F. Bach},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {46--54},
  title = {Data-driven calibration of linear estimators with minimal penalties},
  year = {2010},
}
@inproceedings{kollar10directions,
  author = {T. Kollar and S. Tellex and D. Roy and N. Roy},
  booktitle = {Human-Robot Interaction},
  pages = {259--266},
  title = {Toward Understanding Natural Language Directions},
  year = {2010},
}
@techreport{jaeger08game,
  author = {G. Jäger},
  institution = {University of Tübingen},
  title = {Game Theory in Semantics and Pragmatics},
  year = {2008},
}
@article{jaeger08applications,
  author = {G. Jäger},
  journal = {Language and Linguistics Compass},
  pages = {406--421},
  title = {Applications of Game Theory in Linguistics},
  volume = {2},
  year = {2008},
}
@book{austin62do,
  author = {J. L. Austin},
  publisher = {Oxford},
  title = {How to do Things with Words: The {W}illiam {J}ames Lectures delivered at {H}arvard University in 1955},
  year = {1962},
}
@article{valiant84learnable,
  author = {L. Valiant},
  journal = {Communications of the ACM},
  number = {11},
  pages = {1134--1142},
  title = {A theory of the learnable},
  volume = {27},
  year = {1984},
}
@article{jackson97dnf,
  author = {J. Jackson},
  journal = {Journal of Computer and System Sciences},
  number = {3},
  pages = {414--440},
  title = {An efficient membership-query algorithm for learning {DNF} with respect to the uniform distribution},
  volume = {55},
  year = {1997},
}
@article{angluin88queries,
  author = {D. Angluin},
  journal = {Machine Learning},
  number = {4},
  pages = {319--342},
  title = {Queries and concept learning},
  volume = {2},
  year = {1988},
}
@article{donoho06compressed,
  author = {D. Donoho},
  journal = {IEEE Trans. on Information Theory},
  number = {4},
  pages = {1289--1306},
  title = {Compressed sensing},
  volume = {52},
  year = {2006},
}
@inproceedings{hamlet94random,
  author = {D. Hamlet},
  booktitle = {Encyclopedia of Software Engineering},
  pages = {970--978},
  title = {Random testing},
  year = {1994 1994},
}
@article{graf97predicate,
  author = {S. Graf and H. Saidi},
  journal = {Computer Aided Verification},
  pages = {72--83},
  title = {Construction of abstract state graphs with {PVS}},
  volume = {1254},
  year = {1997},
}
@article{monro51stochastic,
  author = {H. Robbins and S. Monro},
  journal = {Annals of Mathematical Statistics},
  number = {3},
  pages = {400--407},
  title = {A Stochastic Approximation Method},
  volume = {22},
  year = {1951},
}
@inproceedings{smaragdakis11context,
  author = {Y. Smaragdakis and M. Bravenboer and O. Lhotak},
  booktitle = {Principles of Programming Languages (POPL)},
  title = {Pick Your Contexts Well: Understanding Object-Sensitivity},
  year = {2011},
}
@inproceedings{spoon04ddp,
  author = {S. A. Spoon and O. Shivers},
  booktitle = {European Conference on Object-Oriented Programming (ECOOP)},
  title = {Demand-Driven Type Inference with Subgoal Pruning: Trading Precision for Scalability},
  year = {2004},
}
@inproceedings{henzinger02lazy,
  author = {T. A. Henzinger and R. Jhala and R. Majumdar and G. Sutre},
  booktitle = {Principles of Programming Languages (POPL)},
  title = {Lazy Abstraction},
  year = {2002},
}
@inproceedings{mcmillan06lazy,
  author = {K. McMillan},
  booktitle = {Computer Aided Verification (CAV)},
  pages = {123--136},
  title = {Lazy Abstraction with Interpolants},
  year = {2006},
}
@inproceedings{vipindeep05pruning,
  author = {V. Vipindeep and P. Jalote},
  booktitle = {International Workshop on Dynamic Analysis (WODA)},
  title = {Efficient static analysis with path pruning using coverage data},
  year = {2005},
}
@book{kamp93drt,
  author = {H. Kamp and U. Reyle},
  publisher = {Kluwer, Dordrecht},
  title = {From Discourse to Logic: An Introduction to the Model-theoretic Semantics of Natural Language, Formal Logic and Discourse Representation Theory},
  year = {1993},
}
@article{muskens96combine,
  author = {R. Muskens},
  journal = {Linguistics and Philosophy},
  number = {2},
  pages = {143--186},
  title = {Combining Montague semantics and discourse representation},
  volume = {19},
  year = {1996},
}
@inproceedings{kwiatkowski10ccg,
  author = {T. Kwiatkowski and L. Zettlemoyer and S. Goldwater and M. Steedman},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {1223--1233},
  title = {Inducing Probabilistic {CCG} Grammars from Logical Form with Higher-Order Unification},
  year = {2010},
}
@inproceedings{clarke10world,
  author = {J. Clarke and D. Goldwasser and M. Chang and D. Roth},
  booktitle = {Computational Natural Language Learning (CoNLL)},
  pages = {18--27},
  title = {Driving Semantic Parsing from the World's Response},
  year = {2010},
}
@phdthesis{pross09grounded,
  author = {T. Pross},
  school = {University of Stuttgart},
  title = {Grounded Discourse Representation Theory: Towards a Semantics-Pragmatics Interface for Human-Machine Collaboration},
  year = {2009},
}
@techreport{bos94drs,
  author = {J. Bos and E. Mastenbroek and S. McGlashan and S. Millies and M. Pinkal},
  institution = {Universität des Saarlandes},
  title = {A Compositional {DRS}-based formalism for {NLP} applications},
  year = {1994},
}
@incollection{kamp05drt,
  author = {H. Kamp and J. v. Genabith and U. Reyle},
  booktitle = {Handbook of Philosophical Logic},
  title = {Discourse Representation Theory},
  year = {2005},
}
@inproceedings{bos06superlatives,
  author = {M. Nissim and J. Bos},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {An Empirical Approach to the Interpretation of Superlatives},
  year = {2006},
}
@inproceedings{bos09economical,
  author = {J. Bos},
  booktitle = {Workshop on Controlled Natural Language},
  pages = {1--5},
  title = {A Controlled Fragment of {DRT}},
  year = {2009},
}
@inproceedings{bos04wide,
  author = {J. Bos and S. Clark and M. Steedman and J. R. Curran and J. Hockenmaier},
  booktitle = {International Conference on Computational Linguistics (COLING)},
  pages = {1240--1246},
  title = {Wide-coverage semantic representations from a {CCG} parser},
  year = {2004},
}
@article{barker02continuations,
  author = {C. Barker},
  journal = {Natural Language Semantics},
  pages = {211--242},
  title = {Continuations and the nature of quantification},
  volume = {10},
  year = {2002},
}
@techreport{shan04continuations,
  author = {C. Shan},
  institution = {arXiv},
  title = {Delimited continuations in natural language},
  year = {2004},
}
@book{heim98semantics,
  author = {I. Heim and A. Kratzer},
  publisher = {Wiley-Blackwell},
  title = {Semantics in Generative Grammar},
  year = {1998},
}
@article{white06hlds,
  author = {M. White},
  journal = {Research on Language and Computation},
  pages = {39--75},
  title = {Efficient Realization of Coordinate Structures in Combinatory Categorial Grammar},
  volume = {4},
  year = {2006},
}
@inproceedings{bos08framenet,
  author = {J. Bos and M. Nissim},
  booktitle = {Frames, Corpora, and Knowledge Representation},
  pages = {169--183},
  title = {Combining Discourse Representation Theory with FrameNet},
  year = {2008},
}
@inproceedings{chaves07model,
  author = {R. P. Chaves},
  booktitle = {International Workshop on Computational Semantics},
  title = { Dynamic Model Checking of Discourse Representation Structures with Pluralities},
  year = {2007},
}
@book{blackburn05semantics,
  author = {P. Blackburn and J. Bos},
  publisher = {CSLI Publishers},
  title = {Representation and Inference for Natural Language: A First Course in Computational Semantics},
  year = {2005},
}
@inproceedings{thompson97parse,
  author = {C. A. Thompson and R. J. Mooney and L. R. Tang},
  booktitle = {ML-97 Workshop on Automata Induction, Grammatical Inference, and Language Acquisition},
  title = {Learning to Parse Natural Language Database Queries into Logical Form},
  year = {1997},
}
@inproceedings{montague73ptq,
  author = {R. Montague},
  booktitle = {Approaches to Natural Language},
  pages = {221--242},
  title = {The proper treatment of quantification in ordinary {E}nglish},
  year = {1973},
}
@phdthesis{cooper75thesis,
  author = {R. Cooper},
  school = {University of Massachusetts at Amherst},
  title = {Montague's semantic theory and transformational syntax},
  year = {1975},
}
@book{steedman00ccg,
  author = {M. Steedman},
  publisher = {MIT Press},
  title = {The Syntactic Process},
  year = {2000},
}
@book{carpenter98type,
  author = {B. Carpenter},
  publisher = {MIT Press},
  title = {Type-Logical Semantics},
  year = {1998},
}
@inproceedings{zelle96geoquery,
  author = {M. Zelle and R. J. Mooney},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {1050--1055},
  title = {Learning to parse database queries using inductive logic proramming},
  year = {1996},
}
@inproceedings{branavan10high,
  author = {S. Branavan and L. Zettlemoyer and R. Barzilay},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {1268--1277},
  title = {Reading Between the Lines: Learning to Map High-level Instructions to Commands},
  year = {2010},
}
@inproceedings{duchi10adagrad,
  author = {J. Duchi and E. Hazan and Y. Singer},
  booktitle = {Conference on Learning Theory (COLT)},
  title = {Adaptive Subgradient Methods for Online Learning and Stochastic Optimization},
  year = {2010},
  pages={257--269},
}
@book{dechter03constraint,
  author = {R. Dechter},
  publisher = {Morgan Kaufmann},
  title = {Constraint Processing},
  year = {2003},
}
@article{richardson06mln,
  author = {M. Richardson and P. Domingos},
  journal = {Machine Learning},
  pages = {107--136},
  title = {{M}arkov Logic Networks},
  volume = {62},
  year = {2006},
}
@inproceedings{tang01ilp,
  author = {L. R. Tang and R. J. Mooney},
  booktitle = {European Conference on Machine Learning (ECML)},
  pages = {466--477},
  title = {Using multiple clause constructors in inductive logic programming for semantic parsing},
  year = {2001},
}
@inproceedings{kate05funql,
  author = {R. J. Kate and Y. W. Wong and R. J. Mooney},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {1062--1068},
  title = {Learning to Transform Natural to Formal Languages},
  year = {2005},
}
@techreport{nivre05dependency,
  author = {J. Nivre},
  institution = {Växjö University: School of Mathematics and Systems Engineering},
  title = {Dependency Grammar and Dependency Parsing},
  year = {2005},
}
@inproceedings{popescu03precise,
  author = {A. Popescu and O. Etzioni and H. Kautz},
  booktitle = {International Conference on Intelligent User Interfaces (IUI)},
  pages = {149--157},
  title = {Towards a Theory of Natural Language Interfaces to Databases},
  year = {2003},
}
@inproceedings{schuler03interpretation,
  author = {W. Schuler},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {529--536},
  title = {Using model-theoretic semantic interpretation to guide statistical parsing and word recognition in a spoken language interface},
  year = {2003},
}
@article{copestake05mrs,
  author = {A. Copestake and D. Flickinger and I. Sag and C. Pollard},
  journal = {Journal of Research on Language and Computation},
  pages = {281--332},
  title = {Minimal Recursion Semantics: An introduction},
  volume = {3},
  year = {2005},
}
@article{androutsopoulos95nlidb,
  author = {I. Androutsopoulos and G. D. Ritchie and P. Thanisch},
  journal = {Journal of Natural Language Engineering},
  pages = {29--81},
  title = {Natural Language Interfaces to Databases -- An Introduction},
  volume = {1},
  year = {1995},
}
@techreport{woods72lunar,
  author = {W. A. Woods and R. M. Kaplan and B. N. Webber},
  institution = {BBN Report 2378, Bolt Beranek and Newman Inc.},
  title = {The Lunar Sciences Natural Language Information System: Final Report},
  year = {1972},
}
@article{warren82chat80,
  author = {D. Warren and F. Pereira},
  journal = {Computational Linguistics},
  pages = {110--122},
  title = {An Efficient Easily Adaptable System for Interpreting Natural Language Queries},
  volume = {8},
  year = {1982},
}
@inproceedings{baldridge02ccg,
  author = {J. Baldridge and G. M. Kruijff},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {319--326},
  title = {Coupling {CCG} with Hybrid Logic Dependency Semantics},
  year = {2002},
}
@inproceedings{branavan11win,
  author = {S. Branavan and D. Silver and R. Barzilay},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {268--277},
  title = {Learning to Win by Reading Manuals in a {M}onte-{C}arlo Framework},
  year = {2011},
}
@inproceedings{goldwasser11confidence,
  author = {D. Goldwasser and R. Reichart and J. Clarke and D. Roth},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {1486--1495},
  title = {Confidence Driven Unsupervised Semantic Parsing},
  year = {2011},
}
@inproceedings{goldwasser11instructions,
  author = {D. Goldwasser and D. Roth},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  pages = {1794--1800},
  title = {Learning From Natural Instructions},
  year = {2011},
}
@inproceedings{alshawi11nlf,
  author = {H. Alshawi and P. Chang and M. Ringgaard},
  booktitle = {International Conference on Compositional Semantics (IWCS)},
  pages = {15--24},
  title = {Deterministic Statistical Mapping of Sentences to Underspecified Semantics},
  year = {2011},
}
@inproceedings{giordani09sql,
  author = {A. Giordani and A. Moschitti},
  booktitle = {International Conference on Applications of Natural Language to Information Systems},
  pages = {207--221},
  title = {Semantic Mapping Between Natural Language Questions and {SQL} Queries via Syntactic Pairing},
  year = {2009},
}
@inproceedings{miller96statistical,
  author = {S. Miller and D. Stallard and R. Bobrow and R. Schwartz},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {55--61},
  title = {A Fully Statistical Approach to Natural Language Interfaces},
  year = {1996},
}
@inproceedings{kwiatkowski11lex,
  author = {T. Kwiatkowski and L. Zettlemoyer and S. Goldwater and M. Steedman},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {1512--1523},
  title = {Lexical Generalization in {CCG} Grammar Induction for Semantic Parsing},
  year = {2011},
}
@inproceedings{artzi11conversations,
  author = {Y. Artzi and L. Zettlemoyer},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {421--432},
  title = {Bootstrapping Semantic Parsers from Conversations},
  year = {2011},
}
@inproceedings{cousot77abstract,
  author = {P. Cousot and R. Cousot},
  booktitle = {Principles of Programming Languages (POPL)},
  pages = {238--252},
  title = {Abstract interpretation: a unified lattice model for static analysis of programs by construction or approximation of fixpoints},
  year = {1977},
}
@article{nocedal80lbfgs,
  author = {J. Nocedal},
  journal = {Mathematics of Computation},
  pages = {773--782},
  title = {Updating Quasi-Newton Matrices with Limited Storage},
  volume = {35},
  year = {1980},
}
@article{daume09searn,
  author = {H. Daume and J. Langford and D. Marcu},
  journal = {Machine Learning},
  pages = {297--325},
  title = {Search-based Structured Prediction},
  volume = {75},
  year = {2009},
}
@inproceedings{judge06qtb,
  author = {J. Judge and A. Cahill and J. v. Genabith},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  pages = {497--504},
  title = {Question-Bank: creating a corpus of parse-annotated questions},
  year = {2006},
}
@article{porter80stem,
  author = {M. F. Porter},
  journal = {Program: electronic library and information systems},
  pages = {130--137},
  title = {An algorithm for suffix stripping},
  volume = {14},
  year = {1980},
}
@inproceedings{vogel10navigate,
  author = {A. Vogel and D. Jurafsky},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {806--814},
  title = {Learning to Follow Navigational Directions},
  year = {2010},
}
@inproceedings{chen11navigate,
  author = {D. L. Chen and R. J. Mooney},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  pages = {859--865},
  title = {Learning to Interpret Natural Language Navigation Instructions from Observations},
  year = {2011},
}
@article{grice75maxims,
  author = {H. P. Grice},
  journal = {Syntax and semantics},
  pages = {41--58},
  title = {Logic and Conversation},
  volume = {3},
  year = {1975},
}
@phdthesis{potts03thesis,
  author = {C. Potts},
  school = {UC Santa Cruz},
  title = {The Logic of Conventional Implicatures},
  year = {2003},
}
@inproceedings{lev04logic,
  author = {I. Lev and B. MacCartney and C. D. Manning and R. Levy},
  booktitle = {ACL Workshop on Text Meaning and Interpretation},
  title = {Solving Logic Puzzles: From Robust Processing to Precise Semantics},
  year = {2004},
}
@phdthesis{hockenmaier03ccg,
  author = {J. Hockenmaier},
  school = {University of Edinburgh},
  title = {Data and Models for Statistical Parsing with Combinatory Categorial Grammar},
  year = {2003},
}
@inproceedings{holland99proofs,
  author = {A. M. Holland-Minkley and R. Barzilay and R. L. Constable},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Verbalization of High-Level Formal Proofs},
  year = {1999},
}
@inproceedings{dinesh11regulatory,
  author = {N. Dinesh and A. Joshi and I. Lee},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Computing Logical Form on Regulatory Texts},
  year = {2011},
}
@inproceedings{miller04ner,
  author = {S. Miller and J. Guinness and A. Zamanian},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {337--342},
  title = {Name Tagging with Word Clusters and Discriminative Training},
  year = {2004},
}
@inproceedings{koo08simple,
  author = {T. Koo and X. Carreras and M. Collins},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  title = {Simple Semi-Supervised Dependency Parsing},
  year = {2008},
}
@inproceedings{anandkumar12moments,
  author = {A. Anandkumar and D. Hsu and S. M. Kakade},
  booktitle = {Conference on Learning Theory (COLT)},
  title = {A Method of Moments for Mixture Models and Hidden {M}arkov Models},
  year = {2012},
}
@inproceedings{anandkumar11tree,
  author = {A. Anandkumar and K. Chaudhuri and D. Hsu and S. M. Kakade and L. Song and T. Zhang},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Spectral Methods for Learning Multivariate Latent Tree Structure},
  year = {2011},
}
@article{allman11identifiability,
  author = {E. S. Allman and S. Petrovi and J. A. Rhodes and S. Sullivant},
  journal = {Transactions on Computational Biology and Bioinformatics},
  pages = {710--722},
  title = {Identifiability of 2-tree mixtures for group-based models},
  volume = {8},
  year = {2011},
}
@article{allman09identifiability,
  author = {E. S. Allman and C. Matias and J. A. Rhodes},
  journal = {Annals of Statistics},
  pages = {3099--3132},
  title = {Identifiability of parameters in latent structure models with many observed variables},
  volume = {37},
  year = {2009},
}
@article{kruskal77three,
  author = {J. B. Kruskal},
  journal = {Linear Algebra and Applications},
  pages = {95--138},
  title = {Three-way arrays: Rank and uniqueness of trilinear decompositions, with application to arithmetic complexity and statistics},
  volume = {18},
  year = {1977},
}
@inproceedings{hsu09spectral,
  author = {D. Hsu and S. M. Kakade and T. Zhang},
  booktitle = {Conference on Learning Theory (COLT)},
  title = {A spectral algorithm for learning hidden {M}arkov models},
  year = {2009},
}
@article{geiger01stratified,
  author = {D. Geiger and D. Heckerman and H. King and C. Meek},
  journal = {Annals of Statistics},
  pages = {505--529},
  title = {Stratified exponential families: graphical models and model selection},
  volume = {29},
  year = {2001},
}
@article{chi99pcfg,
  author = {Z. Chi},
  journal = {Computational Linguistics},
  title = {Statistical Properties of Probabilistic Context-Free Grammars},
  volume = {25},
  year = {1999},
}
@inproceedings{eisner00cubic,
  author = {J. Eisner},
  booktitle = {Advances in Probabilistic and Other Parsing Technologies},
  pages = {29--62},
  title = {Bilexical grammars and their cubic-time parsing algorithms},
  year = {2000},
}
@inproceedings{etzioni11openie,
  author = {O. Etzioni and A. Fader and J. Christensen and S. Soderland and Mausam},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  title = {Open Information Extraction: the Second Generation},
  year = {2011},
}
@inproceedings{poon10ontology,
  author = {H. Poon and P. Domingos},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Unsupervised Ontology Induction from Text},
  year = {2010},
}
@inproceedings{zhang12smoothing,
  author = {C. Zhang and R. Hoffmann and D. S. Weld},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Ontological Smoothing for Relation Extraction with Minimal Supervision},
  year = {2012},
}
@inproceedings{schoenmackers10horn,
  author = {S. Schoenmackers and O. Etzioni and D. S. Weld and J. Davis},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Learning First-Order Horn Clauses from Web Text},
  year = {2010},
}
@article{jacobs91experts,
  author = {R. A. Jacobs and M. I. Jordan and S. J. Nowlan and G. E. Hinton},
  journal = {Neural Computation},
  pages = {79--87},
  title = {Adaptive mixtures of local experts},
  volume = {3},
  year = {1991},
}
@inproceedings{quattoni04crf,
  author = {A. Quattoni and M. Collins and T. Darrell},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Conditional Random Fields for Object Recognition},
  year = {2004},
}
@inproceedings{petrov08discriminative,
  author = {S. Petrov and D. Klein},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Discriminative Log-Linear Grammars with Latent Variables},
  year = {2008},
}
@inproceedings{wang09crf,
  author = {Y. Wang and G. Mori},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  title = {Max-Margin Hidden Conditional Random Fields for Human Action Recognition},
  year = {2009},
}
@inproceedings{anandkumar12lda,
  author = {A. Anandkumar and D. P. Foster and D. Hsu and S. M. Kakade and Y. Liu},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Two {SVD}s Suffice: Spectral decompositions for probabilistic topic modeling and latent {D}irichlet allocation},
  year = {2012},
}
@inproceedings{hsu13spherical,
  author = {D. Hsu and S. M. Kakade},
  booktitle = {Innovations in Theoretical Computer Science (ITCS)},
  title = {Learning Mixtures of Spherical {G}aussians: Moment Methods and Spectral Decompositions},
  year = {2013},
}
@techreport{candes11phaselift,
  author = {E. J. Candes and T. Strohmer and V. Voroninski},
  institution = {arXiv},
  title = {PhaseLift: Exact and Stable Signal Recovery from Magnitude Measurements via Convex Programming},
  year = {2011},
}
@inproceedings{ohlsson12phase,
  author = {H. Ohlsson and A. Yang and R. Dong and S. Sastry},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {{CPRL} -- An Extension of Compressive Sensing to the Phase Retrieval Problem},
  year = {2012},
}
@article{donoho95soft,
  author = {D. L. Donoho},
  journal = {IEEE Transactions on Information Theory},
  pages = {613--627},
  title = {De-noising by soft-thresholding},
  volume = {41},
  year = {1995},
}
@article{cai10soft,
  author = {J. Cai and E. J. Candes and Z. Shen},
  journal = {SIAM Journal on Optimization},
  pages = {1956--1982},
  title = {A singular value thresholding algorithm for matrix completion},
  volume = {20},
  year = {2010},
}
@inproceedings{barcelo11grpah,
  author = {P. Barcelo and L. Libkin and J. Reutter},
  booktitle = {Symposium on Principles of Database Systems},
  title = {Querying Graph Patterns},
  year = {2011},
}
@inproceedings{lao11random,
  author = {N. Lao and T. Mitchell and W. W. Cohen},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Random Walk Inference and Learning in A Large Scale Knowledge Base},
  year = {2011},
}
@inproceedings{chen12lexicon,
  author = {D. Chen},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Fast Online Lexicon Learning for Grounded Language Acquisition},
  year = {2012},
}
@inproceedings{riedel13universal,
  author = {S. Riedel and L. Yao and A. McCallum},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  title = {Relation Extraction with Matrix Factorization and Universal Schemas},
  year = {2013},
}
@inproceedings{jenatton12latent,
  author = {R. Jenatton and N. L. Roux and A. Bordes and G. Obozinski},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {A latent factor model for highly multi-relational data},
  year = {2012},
}
@inproceedings{nickel12yago,
  author = {M. Nickel and V. Tresp and H. Kriegel},
  booktitle = {World Wide Web (WWW)},
  title = {Factorizing {YAGO}},
  year = {2012},
}
@inproceedings{fader11reverb,
  author = {A. Fader and S. Soderland and O. Etzioni},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Identifying Relations for Open Information Extraction},
  year = {2011},
}
@article{collobert11scratch,
  author = {R. Collobert and J. Weston and L. Bottou and M. Karlen and K. Kavukcuoglu and P. Kuksa},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {2493--2537},
  title = {Natural Language Processing (almost) from Scratch},
  volume = {12},
  year = {2011},
}
@phdthesis{odonnell11fragment,
  author = {T. J. O'Donnell},
  school = {Massachusetts Institute of Technology},
  title = {Productivity and Reuse in Language},
  year = {2011},
}
@inproceedings{song10kernel,
  author = {L. Song and B. Boots and S. Siddiqi and G. Gordon and A. Smola},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {{H}ilbert Space Embeddings of Hidden {M}arkov Models},
  year = {2010},
}
@inproceedings{parikh12spectral,
  author = {A. Parikh and L. Song and M. Ishteva and G. Teodoru and E. Xing},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {A Spectral Algorithm for Latent Junction Trees},
  year = {2012},
}
@inproceedings{cohen12pcfg,
  author = {S. B. Cohen and K. Stratos and M. Collins and D. P. Foster and L. Ungar},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Spectral Learning of Latent-Variable {PCFG}s},
  year = {2012},
}
@inproceedings{balle12automata,
  author = {B. Balle and M. Mohri},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Spectral Learning of General Weighted Automata via Constrained Matrix Completion},
  year = {2012},
}
@inproceedings{balle11transducer,
  author = {B. Balle and A. Quattoni and X. Carreras},
  booktitle = {European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases (ECML-PKDD)},
  title = {A Spectral Learning Algorithm for Finite State Transducers},
  year = {2011},
}
@inproceedings{anandkumar13linear,
  author = {A. Anandkumar and D. Hsu and A. Javanmard and S. M. Kakade},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Learning Linear {B}ayesian Networks with Latent Variables},
  year = {2013},
}
@article{ghahramani97fhmm,
  author = {Z. Ghahramani and M. Jordan},
  journal = {Machine Learning},
  pages = {245--273},
  title = {Factorial Hidden {M}arkov Models},
  volume = {29},
  year = {1997},
}
@techreport{anandkumar13tensor,
  author = {A. Anandkumar and R. Ge and D. Hsu and S. M. Kakade and M. Telgarsky},
  institution = {arXiv},
  title = {Tensor decompositions for learning latent variable models},
  year = {2013},
}
@inproceedings{salakhutdinov09softmax,
  author = {R. Salakhutdinov and G. Hinton},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Replicated Softmax: an Undirected Topic Model},
  year = {2009},
}
@inproceedings{kirkpatrick10painless,
  author = {T. Berg-Kirkpatrick and A. Bouchard-C\^ot\'e and J. DeNero and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  title = {Painless Unsupervised Learning with Features},
  year = {2010},
}
@article{deselaers12latent,
  author = {T. Deselaers and T. Gass and G. Heigold and H. Ney},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {1105--1117},
  title = {Latent log-linear models for handwritten digit classification},
  volume = {34},
  year = {2012},
}
@article{saul96sigmoid,
  author = {L. Saul and T. Jaakkola and M. I. Jordan},
  journal = {Journal of Artificial Intelligence Research (JAIR)},
  pages = {61--76},
  title = {Mean Field Theory for Sigmoid Belief Networks},
  volume = {4},
  year = {1996},
}
@book{smolensky86rbm,
  author = {P. Smolensky},
  pages = {194--281},
  publisher = {MIT Press},
  title = {Parallel Distributed Processing: Volume 1: Foundations},
  year = {1986},
}
@techreport{anandkumar2013overcomplete,
  author = {A. Anandkumar and D. Hsu and M. Janzamin and S. Kakade},
  institution = {arXiv},
  title = {When are Overcomplete Representations Identifiable? Uniqueness of Tensor Decompositions Under Expansion Constraints},
  year = {2013 2013},
}
@inproceedings{manshadi2013integrating,
  author = {M. Manshadi and D. Gildea and J. Allen},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Integrating Programming by Example and Natural Language Programming},
  year = {2013},
}
@inproceedings{halpern2013unsupervised,
  author = {Y. Halpern and D. Sontag},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  title = {Unsupervised Learning of Noisy-Or {B}ayesian Networks},
  year = {2013},
}
@inproceedings{huang2005better,
  author = {L. Huang and D. Chiang},
  booktitle = {Proceedings of the Ninth International Workshop on Parsing Technology},
  pages = {53--64},
  title = {Better k-best parsing},
  year = {2005},
}
@inproceedings{globerson2007fixing,
  author = {A. Globerson and T. Jaakkola},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Fixing max-product: Convergent message passing algorithms for {MAP} {LP}-relaxations},
  year = {2007},
}
@article{kolda2009tensor,
  author = {T. G. Kolda and B. W. Bader},
  journal = {SIAM review},
  number = {3},
  pages = {455--500},
  title = {Tensor decompositions and applications},
  volume = {51},
  year = {2009},
}
@inproceedings{fader2013paraphrase,
  author = {A. Fader and L. Zettlemoyer and O. Etzioni},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Paraphrase-Driven Learning for Open Question Answering},
  year = {2013},
}
@inproceedings{lei2013natural,
  author = {T. Lei and F. Long and R. Barzilay and M. Rinard},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {From Natural Language Specifications to Program Input Parsers},
  year = {2013},
}
@inproceedings{menon2013machine,
  author = {A. Menon and O. Tamuz and S. Gulwani and B. Lampson and A. Kalai},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {A machine learning framework for programming by example},
  year = {2013},
}
@inproceedings{miller2008inky,
  author = {R. C. Miller and V. H. Chou and M. Bernstein and G. Little and M. V. Kleek and D. Karger and m. schraefel},
  booktitle = {User Interface Software and Technology (UIST)},
  pages = {131--140},
  title = {{I}nky: a sloppy command line for the web with rich visual feedback},
  year = {2008},
}
@inproceedings{cai2013large,
  author = {Q. Cai and A. Yates},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Large-scale Semantic Parsing via Schema Matching and Lexicon Extension},
  year = {2013},
}
@inproceedings{yahya2012natural,
  author = {M. Yahya and K. Berberich and S. Elbassuoni and M. Ramanath and V. Tresp and G. Weikum},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {379--390},
  title = {Natural language questions for the web of data},
  year = {2012},
}
@inproceedings{surdeanu2012multi,
  author = {M. Surdeanu and J. Tibshirani and R. Nallapati and C. D. Manning},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {455--465},
  title = {Multi-instance multi-label learning for relation extraction},
  year = {2012},
}
@inproceedings{riedel2010modeling,
  author = {S. Riedel and L. Yao and A. McCallum},
  booktitle = {Machine Learning and Knowledge Discovery in Databases (ECML PKDD)},
  pages = {148--163},
  title = {Modeling relations and their mentions without labeled text},
  year = {2010},
}
@inproceedings{carlson2010toward,
  author = {A. Carlson and J. Betteridge and B. Kisiel and B. Settles and E. R. H. Jr and T. M. Mitchell},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Toward an architecture for never-ending language learning},
  year = {2010},
}
@misc{freebase2013dump,
  author = {Google},
  howpublished = {\url{https://developers.google.com/freebase/data}},
  title = {{F}reebase Data Dumps (2013-06-09)},
  year = {2013},
}
@inproceedings{krishnamurthy2012weakly,
  author = {J. Krishnamurthy and T. Mitchell},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {754--765},
  title = {Weakly supervised training of semantic parsers},
  year = {2012},
}
@inproceedings{masaum2012open,
  author = {Masaum and M. Schmitz and R. Bart and S. Soderland and O. Etzioni},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {523--534},
  title = {Open language learning for information extraction},
  year = {2012},
}
@inproceedings{banko2007open,
  author = {M. Banko and M. J. Cafarella and S. Soderland and M. Broadhead and O. Etzioni},
  booktitle = {International Joint Conference on Artificial Intelligence (IJCAI)},
  pages = {2670--2676},
  title = {Open Information Extraction from the Web},
  year = {2007},
}
@techreport{lacoste2012sigma,
  author = {S. Lacoste-Julien and K. Palla and A. Davies and G. Kasneci and T. Graepel and Z. Ghahramani},
  institution = {arXiv},
  title = {SiGMa: Simple Greedy Matching for Aligning Large Knowledge Bases},
  year = {2012},
}
@inproceedings{auer2007dbpedia,
  author = {S. Auer and C. Bizer and G. Kobilarov and J. Lehmann and R. Cyganiak and Z. G. Ives},
  booktitle = {International semantic web conference and Asian semantic web conference (ISWC/ASWC)},
  pages = {722--735},
  title = {{DB}pedia: A nucleus for a web of open data},
  year = {2007},
}
@inproceedings{lin2012linking,
  author = {T. Lin and Mausam and O. Etzioni},
  booktitle = {Knowledge Extraction Workshop (AKBC-WEKEX)},
  title = {Entity linking at web scale},
  year = {2012},
}
@inproceedings{hearst1992automatic,
  author = {M. A. Hearst},
  booktitle = {Interational Conference on Computational linguistics},
  pages = {539--545},
  title = {Automatic acquisition of hyponyms from large text corpora},
  year = {1992},
}
@inproceedings{chang2012sutime,
  author = {A. X. Chang and C. Manning},
  booktitle = {Language Resources and Evaluation (LREC)},
  pages = {3735--3740},
  title = {{SUT}ime: A library for recognizing and normalizing time expressions},
  year = {2012},
}
@inproceedings{suchanek2007yago,
  author = {F. M. Suchanek and G. Kasneci and G. Weikum},
  booktitle = {World Wide Web (WWW)},
  pages = {697--706},
  title = {{YAGO}: a core of semantic knowledge},
  year = {2007},
}
@inproceedings{kushman2013regex,
  author = {N. Kushman and R. Barzilay},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {826--836},
  title = {Using Semantic Unification to Generate Regular Expressions from Natural Language},
  year = {2013},
}
@article{krishnamurthy2013jointly,
  author = {J. Krishnamurthy and T. Kollar},
  journal = {Transactions of the Association for Computational Linguistics (TACL)},
  pages = {193--206},
  title = {Jointly Learning to Parse and Perceive: Connecting Natural Language to the Physical World},
  volume = {1},
  year = {2013},
}
@article{artzi2013weakly,
  author = {Y. Artzi and L. Zettlemoyer},
  journal = {Transactions of the Association for Computational Linguistics (TACL)},
  pages = {49--62},
  title = {Weakly supervised learning of semantic parsers for mapping instructions to actions},
  volume = {1},
  year = {2013},
}
@book{baader2003description,
  author = {F. Baader},
  publisher = {Cambridge University Press},
  title = {The description logic handbook: theory, implementation, and applications},
  year = {2003},
}
@inproceedings{hoffmann2011knowledge,
  author = {R. Hoffmann and C. Zhang and X. Ling and L. S. Zettlemoyer and D. S. Weld},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {541--550},
  title = {Knowledge-Based Weak Supervision for Information Extraction of Overlapping Relations},
  year = {2011},
}
@inproceedings{finkel2005incorporating,
  author = {J. R. Finkel and T. Grenager and C. Manning},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {363--370},
  title = {Incorporating non-local information into information extraction systems by {G}ibbs sampling},
  year = {2005},
}
@inproceedings{doan2003learning,
  author = {A. Doan and J. Madhavan and R. Dhamankar and P. Domingos and A. Halevy},
  booktitle = {Very Large Data Bases (VLDB)},
  pages = {303--320},
  title = {Learning to match ontologies on the semantic web},
  year = {2003},
}
@inproceedings{poon2013gusp,
  author = {H. Poon},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Grounded Unsupervised Semantic Parsing},
  year = {2013},
}
@inproceedings{tellex2011understanding,
  author = {S. Tellex and T. Kollar and S. Dickerson and M. R. Walter and A. G. Banerjee and S. J. Teller and N. Roy},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Understanding Natural Language Commands for Robotic Navigation and Mobile Manipulation},
  year = {2011},
}
@inproceedings{matuszek2012grounded,
  author = {C. Matuszek and N. FitzGerald and L. Zettlemoyer and L. Bo and D. Fox},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {A joint model of language and perception for grounded attribute learning},
  year = {2012},
}
@inproceedings{socher2012mvrnn,
  author = {R. Socher and B. Huval and C. D. Manning and A. Y. Ng},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {1201--1211},
  title = {Semantic compositionality through recursive matrix-vector spaces},
  year = {2012},
}
@article{yates2009unsupervised,
  author = {A. Yates and O. Etzioni},
  journal = {Journal of Artificial Intelligence Research (JAIR)},
  number = {1},
  title = {Unsupervised methods for determining object and relation synonyms on the web},
  volume = {34},
  year = {2009},
}
@inproceedings{unger2012template,
  author = {C. Unger and L. Bühmann and J. Lehmann and A. Ngonga and D. Gerber and P. Cimiano},
  booktitle = {World Wide Web (WWW)},
  pages = {639--648},
  title = {Template-based question answering over {RDF} data},
  year = {2012},
}
@inproceedings{branavan2012learning,
  author = {S. Branavan and N. Kushman and T. Lei and R. Barzilay},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {126--135},
  title = {Learning high-level planning from text},
  year = {2012},
}
@inproceedings{huang2012wordrep,
  author = {E. H. Huang and R. Socher and C. D. Manning and A. Y. Ng},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Improving Word Representations via Global Context and Multiple Word Prototypes},
  year = {2012},
}
@phdthesis{kushmerick1997wrapper,
  author = {N. Kushmerick},
  school = {University of Washington},
  title = {Wrapper induction for information extraction},
  year = {1997},
}
@inproceedings{cafarella2008webtables,
  author = {M. J. Cafarella and A. Halevy and D. Z. Wang and E. Wu and Y. Zhang},
  booktitle = {Very Large Data Bases (VLDB)},
  pages = {538--549},
  title = {Web{T}ables: exploring the power of tables on the web},
  year = {2008},
}
@article{ferrucci2013watson,
  author = {D. Ferrucci and E. Brown and J. Chu-Carroll and J. Fan and D. Gondek and A. A. Kalyanpur and A. Lally and J. W. Murdock and E. Nyberg and J. Prager and N. Schlaefer and C. Welty},
  journal = {AI Magazine},
  number = {3},
  pages = {59--79},
  title = {Building {W}atson: An Overview of the {D}eep{QA} Project},
  volume = {31},
  year = {2013},
}
@inproceedings{harris2011sparql,
  author = {S. Harris and A. Seaborne},
  booktitle = {W3C Working Draft, 12 May},
  title = {{SPARQL} 1.1 Query Language},
  year = {2011},
}
@inproceedings{clark1975bridging,
  author = {H. H. Clark},
  booktitle = {Workshop on theoretical issues in natural language processing},
  pages = {169--174},
  title = {Bridging},
  year = {1975},
}
@inproceedings{bollacker2008freebase,
  author = {K. Bollacker and C. Evans and P. Paritosh and T. Sturge and J. Taylor},
  booktitle = {International Conference on Management of Data (SIGMOD)},
  pages = {1247--1250},
  title = {{F}reebase: a collaboratively created graph database for structuring human knowledge},
  year = {2008},
}
@inproceedings{weiss2010cascades,
  author = {D. Weiss and B. Taskar},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  title = {Structured Prediction Cascades},
  year = {2010},
}
@inproceedings{kiddon2011coarse,
  author = {C. Kiddon and P. Domingos},
  booktitle = {Association for the Advancement of Artificial Intelligence (AAAI)},
  title = {Coarse-to-Fine Inference and Learning for First-Order Probabilistic Models},
  year = {2011},
}
@inproceedings{mccallum2003efficiently,
  author = {A. McCallum},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  pages = {403--410},
  title = {Efficiently inducing features of conditional random fields},
  year = {2003},
}
@article{della1997inducing,
  author = {S. D. Pietra and V. D. Pietra and J. Lafferty},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  number = {4},
  pages = {380--393},
  title = {Inducing features of random fields},
  volume = {19},
  year = {1997},
}
@inproceedings{ram2011density,
  author = {P. Ram and A. G. Gray},
  booktitle = {International Conference on Knowledge Discovery and Data Mining (KDD)},
  pages = {627--635},
  title = {Density estimation trees},
  year = {2011},
}
@inproceedings{kwiatkowski2013scaling,
  author = {T. Kwiatkowski and E. Choi and Y. Artzi and L. Zettlemoyer},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Scaling Semantic Parsers with On-the-fly Ontology Matching},
  year = {2013},
}
@inproceedings{mintz2009distant,
  author = {M. Mintz and S. Bills and R. Snow and D. Jurafsky},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {1003--1011},
  title = {Distant supervision for relation extraction without labeled data},
  year = {2009},
}
@book{manning2008ir,
  author = {C. Manning and P. Raghavan and H. Schütze},
  publisher = {Cambridge University Press},
  title = {Introduction to information retrieval},
  volume = {1},
  year = {2008},
}
@book{winograd1972language,
  author = {T. Winograd},
  publisher = {Academic Press},
  title = {Understanding Natural Language},
  year = {1972},
}
@inproceedings{dalvi2012websets,
  author = {B. Dalvi and W. Cohen and J. Callan},
  booktitle = {Web Search and Data Mining (WSDM)},
  pages = {243--252},
  title = {WebSets: Extracting sets of entities from the web using unsupervised information extraction},
  year = {2012},
}
@inproceedings{wang2009character,
  author = {R. C. Wang and W. W. Cohen},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {1503--1512},
  title = {Character-level analysis of semi-structured documents for set expansion},
  year = {2009},
}
@article{etzioni2005unsupervised,
  author = {O. Etzioni and M. Cafarella and D. Downey and A. Popescu and T. Shaked and S. Soderland and D. S. Weld and A. Yates},
  journal = {Artificial Intelligence},
  number = {1},
  pages = {91--134},
  title = {Unsupervised named-entity extraction from the web: An experimental study},
  volume = {165},
  year = {2005},
}
@inproceedings{cohen2002flexible,
  author = {W. W. Cohen and M. Hurst and L. S. Jensen},
  booktitle = {World Wide Web (WWW)},
  pages = {232--241},
  title = {A flexible learning system for wrapping tables and lists in {HTML} documents},
  year = {2002},
}
@inproceedings{baroni2010nouns,
  author = {M. Baroni and R. Zamparelli},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {1183--1193},
  title = {Nouns are vectors, adjectives are matrices: Representing adjective-noun constructions in semantic space},
  year = {2010},
}
@inproceedings{ganitkevitch2013ppdb,
  author = {J. Ganitkevitch and B. V. Durme and C. Callison-Burch},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {758--764},
  title = {{PPDB}: The paraphrase database},
  year = {2013},
}
@article{madnani2010generating,
  author = {N. Madnani and B. J. Dorr},
  journal = {Computational Linguistics},
  number = {3},
  pages = {341--387},
  title = {Generating phrasal and sentential paraphrases: A survey of data-driven methods},
  volume = {36},
  year = {2010},
}
@article{androutsopoulos2010survey,
  author = {I. Androutsopoulos and P. Malakasiotis},
  journal = {Journal of Artificial Intelligence Research (JAIR)},
  pages = {135--187},
  title = {A survey of paraphrasing and textual entailment methods},
  volume = {38},
  year = {2010},
}
@inproceedings{heilman2010tree,
  author = {M. Heilman and N. A. Smith},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {1011--1019},
  title = {Tree edit models for recognizing textual entailments, paraphrases, and answers to questions},
  year = {2010},
}
@inproceedings{barzilay2003learning,
  author = {R. Barzilay and L. Lee},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  pages = {16--23},
  title = {Learning to paraphrase: An unsupervised approach using multiple-sequence alignment},
  year = {2003},
}
@inproceedings{chang2005toward,
  author = {K. C. Chang and B. He and Z. Zhang},
  booktitle = {Conference on Innovative Data Systems Research (CIDR)},
  pages = {44--55},
  title = {Toward Large Scale Integration: Building a {M}eta{Q}uerier over Databases on the Web},
  year = {2005},
}
@inproceedings{socher2011paraphrase,
  author = {R. Socher and E. H. Huang and J. Pennin and C. D. Manning and A. Ng},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {801--809},
  title = {Dynamic pooling and unfolding recursive autoencoders for paraphrase detection},
  year = {2011},
}
@article{lin2001discovery,
  author = {D. Lin and P. Pantel},
  journal = {Natural Language Engineering},
  pages = {343--360},
  title = {Discovery of inference rules for question-answering},
  volume = {7},
  year = {2001},
}
@inproceedings{das2009paraphrase,
  author = {D. Das and N. A. Smith},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {468--476},
  title = {Paraphrase identification as probabilistic quasi-synchronous recognition},
  year = {2009},
}
@inproceedings{duclaye2003learning,
  author = {F. Duclaye and F. Yvon and O. Collin},
  booktitle = {Workshop on Natural Language Processing for Question Answering},
  pages = {35--41},
  title = {Learning paraphrases to improve a question-answering system},
  year = {2003},
}
@inproceedings{stern2011transformation,
  author = {A. Stern and I. Dagan},
  booktitle = {Recent Advances in Natural Language Processing},
  pages = {455--462},
  title = {A Confidence Model for Syntactically-Motivated Entailment Proofs},
  year = {2011},
}
@inproceedings{wang2010paraphrasing,
  author = {M. Wang and C. D. Manning},
  booktitle = {The International Conference on Computational Linguistics},
  pages = {1164--1172},
  title = {Probabilistic Tree-Edit Models with Structured Latent Variables for Textual Entailment and Question Answering},
  year = {2010},
}
@inproceedings{malakasiotis2011generate,
  author = {P. Malakasiotis and I. Androutsopoulos},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {96--106},
  title = {A generate and rank approach to sentence paraphrasing},
  year = {2011},
}
@inproceedings{marton2009improved,
  author = {Y. Marton and C. Callison-Burch and P. Resnik},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {381--390},
  title = {Improved statistical machine translation using monolingually-derived paraphrases},
  year = {2009},
}
@inproceedings{zukerman2002lexical,
  author = {I. Zukerman and B. Raskutti},
  booktitle = {International Conference on Computational Linguistics (COLING)},
  pages = {1--7},
  title = {Lexical query paraphrasing for document retrieval},
  year = {2002},
}
@inproceedings{jones2006generating,
  author = {R. Jones and B. Rey and O. Madani and W. Greiner},
  booktitle = {World Wide Web (WWW)},
  pages = {387--396},
  title = {Generating query substitutions},
  year = {2006},
}
@inproceedings{wan2006dependency,
  author = {S. Wan and M. Dras and R. Dale and C. Paris},
  booktitle = {Australasian Language Technology Workshop},
  title = {Using dependency-based features to take the ``para-farce" out of paraphrase},
  year = {2006},
}
@inproceedings{dolan2004unsupervised,
  author = {B. Dolan and C. Quirk and C. Brockett},
  booktitle = {International Conference on Computational Linguistics (COLING)},
  title = {Unsupervised construction of large paraphrase corpora: Exploiting massively parallel news sources},
  year = {2004},
}
@inproceedings{bannard2005paraphrasing,
  author = {C. Bannard and C. Callison-Burch},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {597--604},
  title = {Paraphrasing with bilingual parallel corpora},
  year = {2005},
}
@article{och2004alignment,
  author = {F. J. Och and H. Ney},
  journal = {Computational Linguistics},
  pages = {417--449},
  title = {The Alignment Template Approach to Statistical Machine Translation},
  volume = {30},
  year = {2004},
}
@book{fellbaum1998wordnet,
  author = {C. Fellbaum},
  publisher = {MIT Press},
  title = {WordNet: An Electronic Lexical Database},
  year = {1998},
}
@techreport{mikolov2013efficient,
  author = {T. Mikolov and K. Chen and G. Corrado and Jeffrey},
  institution = {arXiv},
  title = {Efficient Estimation of Word Representations in Vector Space},
  year = {2013},
}
@inproceedings{chang2010discriminative,
  author = {M. Chang and D. Goldwasser and D. Roth and V. Srikumar},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  title = {Discriminative Learning over Constrained Latent Representations},
  year = {2010},
}
@inproceedings{larochelle2008zero,
  author = {H. Larochelle and D. Erhan and Y. Bengio},
  booktitle = {AAAI},
  pages = {646--651},
  title = {Zero-data Learning of New Tasks},
  volume = {8},
  year = {2008},
}
@inproceedings{palatucci2009zero,
  author = {M. Palatucci and D. Pomerleau and G. E. Hinton and T. M. Mitchell},
  booktitle = {Advances in neural information processing systems},
  pages = {1410--1418},
  title = {Zero-shot learning with semantic output codes},
  year = {2009},
}
@inproceedings{liu2000xwrap,
  author = {L. Liu and C. Pu and W. Han},
  booktitle = {Data Engineering, 2000. Proceedings. 16th International Conference on},
  pages = {611--621},
  title = {{XWRAP}: An {XML}-enabled wrapper construction system for web information sources},
  year = {2000},
}
@inproceedings{crescenzi2001roadrunner,
  author = {V. Crescenzi and G. Mecca and P. Merialdo and others},
  booktitle = {VLDB},
  pages = {109--118},
  title = {Roadrunner: Towards automatic data extraction from large web sites},
  volume = {1},
  year = {2001},
}
@article{dalvi2011automatic,
  author = {N. Dalvi and R. Kumar and M. Soliman},
  journal = {Proceedings of the VLDB Endowment},
  number = {4},
  pages = {219--230},
  title = {Automatic wrappers for large scale web extraction},
  volume = {4},
  year = {2011},
}
@inproceedings{freitag1998information,
  author = {D. Freitag},
  booktitle = {AAAI/IAAI},
  pages = {517--523},
  title = {Information extraction from {HTML}: Application of a general machine learning approach},
  year = {1998},
}
@inproceedings{wong2009scalable,
  author = {Y. W. Wong and D. Widdows and T. Lokovic and K. Nigam},
  booktitle = {IEEE International Conference on Data Mining Workshops},
  pages = {302--307},
  title = {Scalable attribute-value extraction from semi-structured text},
  year = {2009},
}
@inproceedings{zhang2013automatic,
  author = {Z. Zhang and K. Q. Zhu and H. Wang and H. Li},
  booktitle = {International Conference on Data Engineering},
  title = {Automatic Extraction of Top-k Lists from the Web},
  year = {2013},
}
@book{fumarola2011extracting,
  author = {F. Fumarola and T. Weninger and R. Barber and D. Malerba and J. Han},
  pages = {285--294},
  publisher = {Modern Approaches in Applied Intelligence Springer},
  title = {Extracting general lists from web documents: A hybrid approach},
  year = {2011},
}
@inproceedings{liu2003mining,
  author = {B. Liu and R. Grossman and Y. Zhai},
  booktitle = {Proceedings of the ninth ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages = {601--606},
  title = {Mining data records in Web pages},
  year = {2003},
}
@inproceedings{zheng2007template,
  author = {S. Zheng and R. Song and J. Wen},
  booktitle = {AAAI},
  pages = {1507--1513},
  title = {Template-independent news extraction based on visual consistency},
  volume = {7},
  year = {2007},
}
@inproceedings{zheng2009efficient,
  author = {S. Zheng and R. Song and J. Wen and C. L. Giles},
  booktitle = {Proceedings of the 18th ACM conference on Information and knowledge management},
  pages = {47--56},
  title = {Efficient record-level wrapper induction},
  year = {2009},
}
@inproceedings{sahuguet1999wysiwyg,
  author = {A. Sahuguet and F. Azavant},
  booktitle = {WWW Conference},
  title = {{WysiWyg} Web Wrapper Factory ({W4F})},
  year = {1999},
}
@inproceedings{toutanova2003tagger,
  author = {K. Toutanova and C. D. Manning},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  title = {Feature-Rich Part-of-Speech Tagging with a Cyclic Dependency Network},
  year = {2003},
}
@techreport{liang2013lambdadcs,
  author = {P. Liang},
  institution = {arXiv},
  title = {Lambda Dependency-Based Compositional Semantics},
  year = {2013},
}
@inproceedings{berant2013freebase,
  author = {J. Berant and A. Chou and R. Frostig and P. Liang},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Semantic Parsing on {F}reebase from Question-Answer Pairs},
  year = {2013},
}
@inproceedings{wang2013noising,
  author = {S. Wang and M. Wang and S. Wager and P. Liang and C. Manning},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Feature Noising for Log-linear Structured Prediction},
  year = {2013},
}
@inproceedings{wager2013dropout,
  author = {S. Wager and S. Wang and P. Liang},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Dropout Training as Adaptive Regularization},
  year = {2013},
}
@inproceedings{chaganty13regression,
  author = {A. Chaganty and P. Liang},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Spectral Experts for Estimating Mixtures of Linear Regressions},
  year = {2013},
}
@inproceedings{ramanathan2013event,
  author = {V. Ramanathan and P. Liang and L. Fei-Fei},
  booktitle = {International Conference on Computer Vision (ICCV)},
  title = {Video Event Understanding using Natural Language Descriptions},
  year = {2013},
}
@inproceedings{sharma13algebraic,
  author = {R. Sharma and S. Gupta and B. Hariharan and A. Aiken and P. Liang and a. A. V. Nori},
  booktitle = {European Symposium on Programming (ESOP)},
  title = {A Data Driven Approach for Algebraic Loop Invariants},
  year = {2013},
}
@inproceedings{hsu12identifiability,
  author = {D. Hsu and S. M. Kakade and P. Liang},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Identifiability and Unmixing of Latent Parse Trees},
  year = {2012},
}
@article{liang13cl,
  author = {P. Liang and M. Jordan and D. Klein},
  journal = {Computational Linguistics},
  pages = {389--446},
  title = {Learning Dependency-Based Compositional Semantics},
  volume = {39},
  year = {2013},
}
@phdthesis{liang11thesis,
  author = {P. Liang},
  school = {University of California Berkeley at Berkeley},
  title = {Learning Dependency-Based Compositional Semantics},
  year = {2011},
}
@inproceedings{liang11dcs,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {590--599},
  title = {Learning Dependency-Based Compositional Semantics},
  year = {2011},
}
@inproceedings{liang11pruning,
  author = {P. Liang and M. Naik},
  booktitle = {Programming Language Design and Implementation (PLDI)},
  title = {Scaling up Abstraction Refinement via Pruning},
  year = {2011},
}
@inproceedings{liang11minimal,
  author = {P. Liang and O. Tripp and M. Naik},
  booktitle = {Principles of Programming Languages (POPL)},
  title = {Learning Minimal Abstractions},
  year = {2011},
}
@inproceedings{golland10pragmatics,
  author = {D. Golland and P. Liang and D. Klein},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {A Game-Theoretic Approach to Generating Spatial Descriptions},
  year = {2010},
}
@inproceedings{angeli10generation,
  author = {G. Angeli and P. Liang and D. Klein},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {A Simple Domain-Independent Probabilistic Approach to Generation},
  year = {2010},
}
@inproceedings{liang10abstraction,
  author = {P. Liang and O. Tripp and M. Naik and M. Sagiv},
  booktitle = {Object-Oriented Programming, Systems, Languages, and Applications (OOPSLA)},
  title = {A Dynamic Evaluation of Static Heap Abstractions},
  year = {2010},
}
@inproceedings{liang10programs,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {639--646},
  title = {Learning Programs: A Hierarchical {B}ayesian Approach},
  year = {2010},
}
@inproceedings{liang10regimes,
  author = {P. Liang and N. Srebro},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {On the Interaction between Norm and Dimensionality: Multiple Regimes in Learning},
  year = {2010},
}
@inproceedings{liang10type,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  title = {Type-Based {MCMC}},
  year = {2010},
}
@techreport{liang10regularizationTR,
  author = {P. Liang and F. Bach and G. Bouchard and M. I. Jordan},
  institution = {arXiv},
  title = {Asymptotically Optimal Regularization in Smooth Parametric Models},
  year = {2010},
}
@inproceedings{liang09regularization,
  author = {P. Liang and F. Bach and G. Bouchard and M. I. Jordan},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Asymptotically Optimal Regularization in Smooth Parametric Models},
  year = {2009},
}
@incollection{liang09hdppcfg,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {The Oxford Handbook of Applied Bayesian Analysis},
  title = {Probabilistic grammars and hierarchical {D}irichlet processes},
  year = {2009},
}
@inproceedings{liang09semantics,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {Association for Computational Linguistics and International Joint Conference on Natural Language Processing (ACL-IJCNLP)},
  pages = {91--99},
  title = {Learning Semantic Correspondences with Less Supervision},
  year = {2009},
}
@inproceedings{liang09measurements,
  author = {P. Liang and M. I. Jordan and D. Klein},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Learning from Measurements in Exponential Families},
  year = {2009},
}
@inproceedings{liang09online,
  author = {P. Liang and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {611--619},
  title = {Online {EM} for Unsupervised Models},
  year = {2009},
}
@inproceedings{liang08asymptotics,
  author = {P. Liang and M. I. Jordan},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {An Asymptotic Analysis of Generative, Discriminative, and Pseudolikelihood Estimators},
  year = {2008},
}
@inproceedings{liang08structure,
  author = {P. Liang and H. Daume and D. Klein},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Structure Compilation: Trading Structure for Features},
  year = {2008},
}
@inproceedings{liang08errors,
  author = {P. Liang and D. Klein},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  title = {Analyzing the Errors of Unsupervised Learning},
  year = {2008},
}
@inproceedings{haghighi08lexicon,
  author = {A. Haghighi and P. Liang and T. Berg-Kirkpatrick and D. Klein},
  booktitle = {Human Language Technology and Association for Computational Linguistics (HLT/ACL)},
  title = {Learning Bilingual Lexicons from Monolingual Corpora},
  year = {2008},
}
@inproceedings{liang08agreement,
  author = {P. Liang and D. Klein and M. I. Jordan},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Agreement-Based Learning},
  year = {2008},
}
@inproceedings{bouchard08language,
  author = {A. Bouchard-C\^ot\'e and P. Liang and T. Griffiths and D. Klein},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {A Probabilistic Approach to Language Change},
  year = {2008},
}
@inproceedings{liang07tutorial,
  author = {P. Liang and D. Klein},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Structured {B}ayesian Nonparametric Models with Variational Inference (tutorial)},
  year = {2007},
}
@inproceedings{liang07permdp,
  author = {P. Liang and M. I. Jordan and B. Taskar},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {A permutation-augmented sampler for {D}irichlet process mixture models},
  year = {2007},
}
@inproceedings{liang07infpcfg,
  author = {P. Liang and S. Petrov and M. I. Jordan and D. Klein},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  title = {The Infinite {PCFG} using Hierarchical {D}irichlet Processes},
  year = {2007},
}
@inproceedings{bouchard07diachronic,
  author = {A. Bouchard-C\^ot\'e and P. Liang and T. Griffiths and D. Klein},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  title = {A Probabilistic Approach to Diachronic Phonology},
  year = {2007},
}
@inproceedings{liang06discrimative,
  author = {P. Liang and A. Bouchard-C\^ot\'e and D. Klein and B. Taskar},
  booktitle = {International Conference on Computational Linguistics and Association for Computational Linguistics (COLING/ACL)},
  title = {An End-to-End Discriminative Approach to Machine Translation},
  year = {2006},
}
@inproceedings{liang06alignment,
  author = {P. Liang and B. Taskar and D. Klein},
  booktitle = {North American Association for Computational Linguistics (NAACL)},
  pages = {104--111},
  title = {Alignment by Agreement},
  year = {2006},
}
@mastersthesis{liang05meng,
  author = {P. Liang},
  school = {Massachusetts Institute of Technology},
  title = {Semi-Supervised Learning for Natural Language},
  year = {2005},
}
@techreport{liang05hypercycle,
  author = {P. Liang and N. Srebro},
  institution = {Massachusetts Institute of Technology},
  title = {A Data Structure for Maintaining Acyclicity in Hypergraphs},
  year = {2005},
}
@techreport{liang04markov,
  author = {P. Liang and N. Srebro},
  institution = {Massachusetts Institute of Technology},
  title = {Methods and Experiments With Bounded Tree-width {M}arkov Networks},
  year = {2004},
}
@techreport{liang03maxwmfarm,
  author = {P. Liang and N. Srebro},
  institution = {Massachusetts Institute of Technology},
  title = {How Much Of A Hypertree Can Be Captured By Windmills?},
  year = {2003},
}
@inproceedings{liang05mcmaster,
  author = {P. Liang and N. Srebro},
  booktitle = {Mathematical Programing for Data Mining and Machine Learning Workshop at McMaster University},
  title = {Linear Programming in Bounded Tree-width {M}arkov Networks},
  year = {2005},
}
@inproceedings{liang05geometric,
  author = {P. Liang and M. Narasimhan and M. Shilman and P. Viola},
  booktitle = {International Conference on Document Analysis and Recognition (ICDAR)},
  title = {Efficient Geometric Algorithms for Parsing in Two Dimensions},
  year = {2005},
}
@book{dagan2013rte,
  author = {I. Dagan and D. Roth and M. Sammons and F. M. Zanzotto},
  publisher = {Morgan and Claypool Publishers},
  title = {Recognizing Textual Entailment: Models and Applications},
  year = {2013},
}
@inproceedings{harabagiu06methodsfor,
  author = {S. Harabagiu and A. Hickl},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Methods for Using Textual Entailment in Open-Domain Question Answering},
  year = {2006},
}
@inproceedings{romano2006investigating,
  author = {L. Romano and M. kouylekov and I. Szpektor and I. Dagan and A. Lavelli},
  booktitle = {European Association for Computational Linguistics (EACL)},
  title = {Investigating a Generic Paraphrase-based Approach for Relation Extraction},
  year = {2006},
}
@article{piwek2012varieties,
  author = {P. Piwek and K. E. Boyer},
  journal = {Dialogue and Discourse},
  pages = {1--9},
  title = {Varieties of Question Generation: Introduction to this Special Issue},
  volume = {3},
  year = {2012},
}
@article{yao2012semantics,
  author = {X. Yao and G. Bouma and Y. Zhang},
  journal = {Dialogue and Discourse},
  pages = {11--42},
  title = {Semantics-based Question Generation and Implementation},
  volume = {3},
  year = {2012},
}
@article{olney2012question,
  author = {A. M. Olney and A. C. Graesser and N. K. Person},
  journal = {Dialogue and Discourse},
  pages = {75--99},
  title = {Question Generation from Concept Maps},
  volume = {3},
  year = {2012},
}
@article{muslea2001hierarchical,
  author = {I. Muslea and S. Minton and C. A. Knoblock},
  journal = {Autonomous Agents and Multi-Agent Systems},
  number = {1},
  pages = {93--114},
  title = {Hierarchical wrapper induction for semistructured information sources},
  volume = {4},
  year = {2001},
}
@inproceedings{haghighi05robust,
  author = {A. Haghighi and A. Y. Ng and C. D. Manning},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  title = {Robust Textual Inference via Graph Matching},
  year = {2005},
}
@inproceedings{spitkovsky2012cross,
  author = {V. I. Spitkovsky and A. X. Chang},
  booktitle = {Language Resources and Evaluation (LREC)},
  pages = {3168--3175},
  title = {A Cross-Lingual Dictionary for {E}nglish {W}ikipedia Concepts},
  year = {2012},
}
@inproceedings{bergsma2008discriminative,
  author = {S. Bergsma and D. Lin and R. Goebel},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {59--68},
  title = {Discriminative learning of selectional preference from unlabeled text},
  year = {2008},
}
@article{recasens2010paraphrase,
  author = {M. Recasens and M. Vila},
  journal = {Computational Linguistics},
  number = {4},
  pages = {639--647},
  title = {On paraphrase and coreference},
  volume = {36},
  year = {2010},
}
@inproceedings{singh2011large,
  author = {S. Singh and A. Subramanya and F. Pereira and A. McCallum},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {793--803},
  title = {Large-scale cross-document coreference using distributed inference and hierarchical models},
  year = {2011},
}
@inproceedings{snow2004learning,
  author = {R. Snow and D. Jurafsky and A. Y. Ng},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Learning syntactic patterns for automatic hypernym discovery},
  year = {2004},
}
@inproceedings{pauls2009kbest,
  author = {A. Pauls and D. Klein},
  booktitle = {Association for Computational Linguistics (ACL)},
  pages = {958--966},
  title = {{K}-best {A*} parsing},
  year = {2009},
}
@inproceedings{klein2002fast,
  author = {D. Klein and C. D. Manning},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Fast exact inference with a factored model for natural language parsing},
  year = {2002},
}
@inproceedings{song2011kernel,
  author = {L. Song and E. P. Xing and A. P. Parikh},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {2708--2716},
  title = {Kernel embeddings of latent tree graphical models},
  year = {2011},
}
@mastersthesis{anca2009math,
  author = {S. Anca},
  school = {Jacobs University Bremen},
  title = {Natural Language and Mathematics Processing for Applicable Theorem Search},
  year = {2009},
}
@book{ganesalingam2013math,
  author = {M. Ganesalingam},
  publisher = {Springer-Verlag},
  title = {The Language of Mathematics},
  year = {2013},
}
@inproceedings{klein2003fast,
  author = {D. Klein and C. Manning},
  booktitle = {Human Language Technology and North American Association for Computational Linguistics (HLT/NAACL)},
  title = {{A*} Parsing: Fast Exact Viterbi Parse Selection},
  year = {2003},
}
@inproceedings{zhu2005twod,
  author = {J. Zhu and Z. Nie and J. Wen and B. Zhang and W. Ma},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {1044--1051},
  title = {2{D} conditional random fields for web information extraction},
  year = {2005},
}
@inproceedings{song2004learning,
  author = {R. Song and H. Liu and J. Wen and W. Ma},
  booktitle = {World Wide Web (WWW)},
  pages = {203--211},
  title = {Learning block importance models for web pages},
  year = {2004},
}
@inproceedings{arasu2003extracting,
  author = {A. Arasu and H. Garcia-Molina},
  booktitle = {ACM SIGMOD international conference on Management of data},
  pages = {337--348},
  title = {Extracting structured data from web pages},
  year = {2003},
}
@book{siefkes2005incremental,
  author = {C. Siefkes},
  pages = {510--521},
  publisher = {Computational Linguistics and Intelligent Text Processing Springer},
  title = {Incremental information extraction using tree-based context representations},
  year = {2005},
}
@article{andrieu2010particle,
  author = {C. Andrieu and A. Doucet and R. Holenstein},
  journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)},
  number = {3},
  pages = {269--342},
  title = {Particle {M}arkov chain {M}onte {C}arlo methods},
  volume = {72},
  year = {2010},
}
@book{bertot2004interactive,
  author = {Y. Bertot and P. Castéran},
  publisher = {Springer},
  title = {Interactive theorem proving and program development: {C}oq'{A}rt: the calculus of inductive constructions},
  year = {2004},
}
@inproceedings{taskar03maxmargin,
  author = {B. Taskar and C. Guestrin and D. Koller},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Max-margin {M}arkov Networks},
  year = {2003},
}
@article{jaeger2000observable,
  author = {H. Jaeger},
  journal = {Neural Computation},
  number = {6},
  pages = {1371--1398},
  title = {Observable operator models for discrete stochastic time series},
  volume = {12},
  year = {2000},
}
@article{liang2014semantics,
  author = {P. Liang and C. Potts},
  journal = {Annual Reviews of Linguistics (submitted)},
  title = {Bringing machine learning and compositional semantics together},
  volume = {0},
  year = {2014},
}
@inproceedings{chaganty2014graphical,
  author = {A. Chaganty and P. Liang},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Estimating Latent-Variable Graphical Models using Moments and Likelihoods},
  year = {2014},
}
@inproceedings{steinhardt2014eg,
  author = {J. Steinhardt and P. Liang},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Adaptivity and Optimism: An Improved Exponentiated Gradient Algorithm},
  year = {2014},
}
@inproceedings{berant2014paraphrasing,
  author = {J. Berant and P. Liang},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Semantic Parsing via Paraphrasing},
  year = {2014},
}
@inproceedings{pasupat2014extraction,
  author = {P. Pasupat and P. Liang},
  booktitle = {Association for Computational Linguistics (ACL)},
  title = {Zero-shot Entity Extraction from Web Pages},
  year = {2014},
}
@inproceedings{wang2014iqp,
  author = {S. Wang and R. Frostig and P. Liang and C. Manning},
  booktitle = {International Conference on Learning Representations (ICLR)},
  title = {Relaxations for inference in restricted {B}oltzmann machines},
  year = {2014},
}
@inproceedings{steinhardt2014filtering,
  author = {J. Steinhardt and P. Liang},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Filtering with Abstract Particles},
  year = {2014},
}
@inproceedings{ghahramani1999variational,
  author = {Z. Ghahramani and M. J. Beal},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Variational Inference for {B}ayesian Mixtures of Factor Analysers.},
  year = {1999},
}
@article{jaakkola1999variational,
  author = {T. S. Jaakkola and M. I. Jordan},
  journal = {Journal of Artificial Intelligence Research (JAIR)},
  pages = {291--322},
  title = {Variational Probabilistic Inference and the {QMR-DT} Network},
  volume = {10},
  year = {1999},
}
@inproceedings{mossel2005learning,
  author = {E. Mossel and S. Roch},
  booktitle = {Theory of computing},
  pages = {366--375},
  title = {Learning nonsingular phylogenies and hidden {M}arkov models},
  year = {2005},
}
@inproceedings{bailly2010spectral,
  author = {R. Bailly and A. Habrard and F. Denis},
  booktitle = {Algorithmic Learning Theory},
  pages = {74--88},
  title = {A spectral approach for probabilistic grammatical inference on trees},
  year = {2010},
}
@inproceedings{song2011spectral,
  author = {L. Song and E. P. Xing and A. P. Parikh},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {A spectral algorithm for latent tree graphical models},
  year = {2011},
}
@book{koller2009probabilistic,
  author = {D. Koller and N. Friedman},
  publisher = {MIT Press},
  title = {Probabilistic graphical models: principles and techniques},
  year = {2009},
}
@article{quattoni2007hidden,
  author = {A. Quattoni and S. Wang and L. Morency and M. Collins and T. Darrell},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  number = {10},
  pages = {1848--1852},
  title = {Hidden-state conditional random fields},
  volume = {29},
  year = {2007},
}
@inproceedings{ross2011learning,
  author = {S. Ross and D. Munoz and M. Hebert and J. A. Bagnell},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  pages = {2737--2744},
  title = {Learning message-passing inference machines for structured prediction},
  year = {2011},
}
@article{bunse1993numerical,
  author = {A. Bunse-Gerstner and R. Byers and V. Mehrmann},
  journal = {SIAM Journal on Matrix Analysis and Applications},
  number = {4},
  pages = {927--949},
  title = {Numerical methods for simultaneous diagonalization},
  volume = {14},
  year = {1993},
}
@article{chandrasekaran2013computational,
  author = {V. Chandrasekaran and M. I. Jordan},
  journal = {Proceedings of the National Academy of Sciences},
  number = {13},
  pages = {1181--1190},
  title = {Computational and statistical tradeoffs via convex relaxation},
  volume = {110},
  year = {2013},
}
@article{shulaker2013carbon,
  author = {M. M. Shulaker and G. Hills and N. Patil and H. Wei and H. Chen and H. P. Wong and S. Mitra},
  journal = {Nature},
  number = {7468},
  pages = {526--530},
  title = {Carbon nanotube computer},
  volume = {501},
  year = {2013},
}
@article{choi2007low,
  author = {J. W. Choi and B. Shim and A. C. Singer and N. I. Cho},
  journal = {IEEE Transactions on Signal Processing},
  number = {10},
  pages = {5084--5096},
  title = {Low-power filtering via minimum power soft error cancellation},
  volume = {55},
  year = {2007},
}
@article{schnaufer1997adaptive,
  author = {B. A. Schnaufer and W. Jenkins},
  journal = {IEEE Transactions on Circuits and Systems},
  number = {12},
  pages = {1001--1014},
  title = {Adaptive fault tolerance for reliable {LMS} adaptive filtering},
  volume = {44},
  year = {1997},
}
@article{zhang2008bibliographical,
  author = {Y. Zhang and J. Jiang},
  journal = {Annual reviews in control},
  number = {2},
  pages = {229--252},
  title = {Bibliographical review on reconfigurable fault-tolerant control systems},
  volume = {32},
  year = {2008},
}
@inproceedings{lin2010energy,
  author = {K. Lin and A. Kansal and D. Lymberopoulos and F. Zhao},
  booktitle = {International conference on Mobile systems, applications, and services},
  pages = {285--298},
  title = {Energy-accuracy trade-off for continuous mobile device location},
  year = {2010},
}
@techreport{shamir2013fundamental,
  author = {O. Shamir},
  institution = {arXiv},
  title = {Fundamental Limits of Online and Distributed Algorithms for Statistical Learning and Estimation},
  year = {2013},
}
@techreport{chen2014statistical,
  author = {Y. Chen and J. Xu},
  institution = {arXiv},
  title = {Statistical-Computational Tradeoffs in Planted Problems and Submatrix Localization with a Growing Number of Clusters and Submatrices},
  year = {2014},
}
@inproceedings{shivakumar2002modeling,
  author = {P. Shivakumar and M. Kistler and S. W. Keckler and D. Burger and L. Alvisi},
  booktitle = {International Conference on Dependable Systems and Networks (DSN)},
  pages = {389--398},
  title = {Modeling the effect of technology trends on the soft error rate of combinational logic},
  year = {2002},
}
@article{sacks1958asymptotic,
  author = {J. Sacks},
  journal = {Annals of Mathematical Statistics},
  number = {2},
  pages = {373--405},
  title = {Asymptotic Distribution of Stochastic Approximation Procedures},
  volume = {29},
  year = {1958},
}
@article{gladyshev1965stochastic,
  author = {G. E. Gladyshev},
  journal = {Theory of Probability and its Applications},
  number = {2},
  pages = {275--278},
  title = {On Stochastic Approximation},
  volume = {10},
  year = {1965},
}
@article{fabian1968asymptotic,
  author = {V. Fabian},
  journal = {Annals of Mathematical Statistics},
  number = {4},
  pages = {1327--1332},
  title = {On Aymptotic Normality in Stochastic Approximation},
  volume = {39},
  year = {1968},
}
@article{chung1954stochastic,
  author = {K. L. Chung},
  journal = {Annals of Mathematical Statistics},
  number = {3},
  pages = {463--483},
  title = {On a Stochastic Approximation Method},
  volume = {25},
  year = {1954},
}
@article{blum1954multidimensional,
  author = {J. R. Blum},
  journal = {Annals of Mathematical Statistics},
  number = {4},
  pages = {737--744},
  title = {Multidimensional Stochastic Approximation Methods},
  volume = {25},
  year = {1954},
}
@techreport{hinton2010practical,
  author = {G. Hinton},
  institution = {University of Toronto},
  title = {A practical guide to training restricted {B}oltzmann machines},
  year = {2010},
}
@inproceedings{salakhutdinov2008ais,
  author = {R. Salakhutdinov and I. Murray},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {872--879},
  title = {On the quantitative analysis of deep belief networks},
  year = {2008},
}
@inproceedings{long2010restricted,
  author = {P. Long and R. Servedio},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {703--710},
  title = {Restricted {B}oltzmann machines are hard to approximately evaluate or simulate},
  year = {2010},
}
@inproceedings{ravikumar2006quadratic,
  author = {P. Ravikumar and J. Lafferty},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {737--744},
  title = {Quadratic programming relaxations for metric labeling and {M}arkov random field {MAP} estimation},
  year = {2006},
}
@article{alon2006approximating,
  author = {N. Alon and A. Naor},
  journal = {SIAM Journal on Computing},
  number = {4},
  pages = {787--803},
  title = {Approximating the cut-norm via Grothendieck's inequality},
  volume = {35},
  year = {2006},
}
@article{goemans1995improved,
  author = {M. Goemans and D. Williamson},
  journal = {Journal of the ACM (JACM)},
  number = {6},
  pages = {1115--1145},
  title = {Improved approximation algorithms for maximum cut and satisfiability problems using semidefinite programming},
  volume = {42},
  year = {1995},
}
@article{pataki1998rank,
  author = {G. Pataki},
  journal = {Mathematics of Operations Research},
  number = {2},
  pages = {339--358},
  title = {On the rank of extreme matrices in semidefinite programs and the multiplicity of optimal eigenvalues},
  volume = {23},
  year = {1998},
}
@article{burer01anonlinear,
  author = {S. Burer and R. Monteiro},
  journal = {Mathematical Programming (series B)},
  title = {A Nonlinear Programming Algorithm for Solving Semidefinite Programs via Low-rank Factorization},
  volume = {95},
  year = {2001},
}
@article{burer2005local,
  author = {S. Burer and R. Monteiro},
  journal = {Mathematical Programming},
  number = {3},
  pages = {427--444},
  title = {Local minima and convergence in low-rank semidefinite programming},
  volume = {103},
  year = {2005},
}
@article{nesterov1998semidefinite,
  author = {Y. Nesterov},
  journal = {Optimization methods and software},
  pages = {141--160},
  title = {Semidefinite relaxation and nonconvex quadratic optimization},
  volume = {9},
  year = {1998},
}
@article{rietz1974proof,
  author = {R. E. Rietz},
  journal = {Israel Journal of Mathematics},
  number = {3},
  pages = {271--276},
  title = {A proof of the {G}rothendieck inequality},
  volume = {19},
  year = {1974},
}
@inproceedings{briet2010positive,
  author = {J. Bri{\"e}t and F. M. d. O. Filho and F. Vallentin},
  booktitle = {Automata, Languages and Programming},
  pages = {31--42},
  title = {The positive semidefinite {G}rothendieck problem with rank constraint},
  year = {2010},
}
@article{malik06thegap,
  author = {U. Malik and I. M. Jaimoukha and G. D. Halikias and S. K. Gungah},
  journal = {Mathematical Programming},
  number = {3},
  pages = {505--515},
  title = {On the gap between the quadratic integer programming problem and its semidefinite relaxation},
  volume = {107},
  year = {2006},
}
@article{halikias07newbounds,
  author = {G. D. Halikias and I. M. Jaimoukha and U. Malik and S. K. Gungah},
  journal = {Journal of Global Optimization},
  number = {4},
  pages = {543--554},
  title = {New bounds on the unconstrained quadratic integer programming problem.},
  volume = {39},
  year = {2007},
}
@article{barvinok95problems,
  author = {A. I. Barvinok},
  journal = {Discrete \& Computational Geometry},
  pages = {189--202},
  title = {Problems of Distance Geometry and Convex Properties of Quadratic Maps.},
  volume = {13},
  year = {1995},
}
@inproceedings{riedel2012parse,
  author = {S. Riedel and D. Smith and A. McCallum},
  booktitle = {Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP/CoNLL)},
  pages = {732--743},
  title = {Parse, price and cut: delayed column and row generation for graph based parsers},
  year = {2012},
}
@article{shi2000normalized,
  author = {J. Shi and J. Malik},
  journal = {Pattern Analysis and Machine Intelligence, IEEE Transactions on},
  number = {8},
  pages = {888--905},
  title = {Normalized cuts and image segmentation},
  volume = {22},
  year = {2000},
}
@inproceedings{kulis2007fast,
  author = {B. Kulis and A. C. Surendran and J. C. Platt},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  pages = {235--242},
  title = {Fast low-rank semidefinite programming for embedding and clustering},
  year = {2007},
}
@article{kumar2009analysis,
  author = {M. P. Kumar and V. Kolmogorov and P. Torr},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {71--106},
  title = {An analysis of convex relaxations for {MAP} estimation of discrete {MRF}s},
  volume = {10},
  year = {2009},
}
@article{recht2013parallel,
  author = {B. Recht and C. R{'e}},
  journal = {Mathematical Programming Computation},
  pages = {1--26},
  title = {Parallel stochastic gradient algorithms for large-scale matrix completion},
  volume = {5},
  year = {2013},
}
@inproceedings{lee2010practical,
  author = {J. Lee and B. Recht and N. Srebro and J. Tropp and R. Salakhutdinov},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {1297--1305},
  title = {Practical large-scale optimization for max-norm regularization},
  year = {2010},
}
@inproceedings{krahenbuhl2011efficient,
  author = {P. Kr{\"a}henb{\"u}hl and V. Koltun},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  title = {Efficient inference in fully connected {CRF}s with {G}aussian edge potentials},
  year = {2011},
}
@inproceedings{krahenbuhl2013learning,
  author = {P. Kr{\"a}henb{\"u}hl and V. Koltun},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {513--521},
  title = {Parameter Learning and Convergent Inference for Dense Random Fields},
  year = {2013},
}
@inproceedings{rush2010dual,
  author = {A. Rush and D. Sontag and M. Collins and T. Jaakkola},
  booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
  pages = {1--11},
  title = {On dual decomposition and linear programming relaxations for natural language processing},
  year = {2010},
}
@inproceedings{sontag2008outer,
  author = {D. Sontag and T. Jaakkola},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {1393--1400},
  title = {New Outer Bounds on the Marginal Polytope},
  year = {2008},
}
@inproceedings{sontag2008tightening,
  author = {D. Sontag and T. Meltzer and A. Globerson and Y. Weiss and T. Jaakkola},
  booktitle = {Uncertainty in Artificial Intelligence (UAI)},
  pages = {503--510},
  title = {Tightening {LP} Relaxations for {MAP} using Message-Passing},
  year = {2008},
}
@inproceedings{cour2007solving,
  author = {T. Cour and J. Shi},
  booktitle = {Artificial Intelligence and Statistics (AISTATS)},
  pages = {75--82},
  title = {Solving {M}arkov random fields with spectral relaxation},
  year = {2007},
}
@article{winder66partition,
  author = {R. O. Winder},
  journal = {SIAM Journal on Applied Mathematics},
  number = {4},
  pages = {811--818},
  title = {Partitions of {N}-Space by Hyperplanes},
  volume = {14},
  year = {1966},
}
@inproceedings{kappes2013benchmark,
  author = {J. H. Kappes and B. Andres and F. A. Hamprecht and C. Schn{\"o}rr and S. Nowozin and D. Batra and S. Kim and B. X. Kausler and J. Lellmann and N. Komodakis and C. Rother},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  title = {A Comparative Study of Modern Inference Techniques for Discrete Energy Minimization Problem},
  year = {2013},
}
@article{allemand2001polynomial,
  author = {K. Allemand and K. Fukuda and T. M. Liebling and E. Steiner},
  journal = {Mathematical programming},
  number = {1},
  pages = {49--52},
  title = {A polynomial case of unconstrained zero-one quadratic optimization},
  volume = {91},
  year = {2001},
}
@article{geman1984stochastic,
  author = {S. Geman and D. Geman},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {721--741},
  title = {Stochastic relaxation, {G}ibbs distributions, and the {B}ayesian restoration of images},
  volume = {6},
  year = {1984},
}
@inproceedings{hills2013rapid,
  author = {G. Hills and J. Zhang and C. Mackin and M. Shulaker and H. Wei and H. S. P. Wong and S. Mitra},
  booktitle = {Proceedings of the 50th Annual Design Automation Conference},
  title = {Rapid Exploration of Processing and Design Guidelines to Overcome Carbon Nanotube Variations},
  year = {2013},
}
@article{lohstroh1983worst,
  author = {J. Lohstroh and E. Seevinck and J. D. Groot},
  journal = {IEEE Journal of Solid-State Circuits},
  number = {6},
  pages = {803--807},
  title = {Worst-case static noise margin criteria for logic circuits and their mathematical equivalence},
  volume = {18},
  year = {1983},
}
@article{luo2013compact,
  author = {J. Luo and L. Wei and C. Lee and A. D. Franklin and X. Guan and E. Pop and D. Antoniadis and H. S. P. Wong},
  journal = {IEEE Transactions on Electron Devices},
  number = {6},
  pages = {1834--1843},
  title = {Compact Model for Carbon Nanotube Field-Effect Transistors Including Nonidealities and Calibrated with Experimental Data Down to 9-nm Gate Length},
  volume = {60},
  year = {2013},
}
@article{cardoso1996joint,
  author = {J. Cardoso and A. Souloumiac},
  journal = {SIAM Journal on Matrix Analysis and Applications},
  number = {1},
  pages = {161--164},
  title = {Jacobi angles for simultaneous diagonalization},
  volume = {17},
  year = {1996},
}
@article{afsari2008sensitivity,
  author = {B. Afsari},
  journal = {SIAM Journal on Matrix Analysis and Applications},
  number = {3},
  pages = {1148--1171},
  title = {Sensitivity analysis for the problem of matrix joint diagonalization},
  volume = {30},
  year = {2008},
}
@techreport{cardoso1994perturbation,
  author = {J. Cardoso},
  institution = {T'{e}l'{e}com Paris},
  title = {Perturbation of Joint Diagonalizers},
  year = {1994},
}
@article{laurent2000adaptive,
  author = {B. Laurent and P. Massart},
  journal = {Annals of Statistics},
  number = {5},
  pages = {1302--1338},
  title = {Adaptive estimation of a quadratic functional by model selection},
  volume = {28},
  year = {2000},
}
@article{hastad1990tensor,
  author = {J. Ho{a}stad},
  journal = {Journal of Algorithms},
  number = {4},
  title = {Tensor rank is {NP}-complete},
  volume = {11},
  year = {1990},
}
@inproceedings{souloumiac2009joint,
  author = {S. A.},
  booktitle = {Computational Advances in Multi-Sensor Adaptive Processing},
  pages = {305--308},
  title = {Joint diagonalization: Is non-orthogonal always preferable to orthogonal?},
  year = {2009},
}
@article{delathauwer2006decomposition,
  author = {L. D. Lathauwer},
  journal = {SIAM Journal of Matrix Analysis and Applications},
  number = {3},
  pages = {642--666},
  title = {A Link Between the Canonical Decomposition in Multilinear Algebra and Simultaneous Matrix Diagonalization},
  volume = {28},
  year = {2006},
}
@inproceedings{vasilescu2005multilinear,
  author = {M. A. O. Vasilescu and D. Terzopoulos},
  booktitle = {Computer Vision and Pattern Recognition (CVPR)},
  pages = {547--553},
  title = {Multilinear independent components analysis},
  volume = {1},
  year = {2005},
}
@article{beckmann2005tensorial,
  author = {C. F. Beckmann and S. M. Smith},
  journal = {Neuroimage},
  number = {1},
  pages = {294--311},
  title = {Tensorial extensions of independent component analysis for multisubject {FMRI} analysis},
  volume = {25},
  year = {2005},
}
@article{yeredor2002non,
  author = {A. Yeredor},
  journal = {IEEE Transactions on Signal Processing},
  number = {7},
  pages = {1545--1553},
  title = {Non-orthogonal joint diagonalization in the least-squares sense with application in blind source separation},
  volume = {50},
  year = {2002},
}
@article{ziehe2004fast,
  author = {A. Ziehe and P. Laskov and G. Nolte and K. M{\"u}ller},
  journal = {Journal of Machine Learning Research (JMLR)},
  pages = {777--800},
  title = {A fast algorithm for joint diagonalization with non-orthogonal transformations and its application to blind source separation},
  volume = {5},
  year = {2004},
}
@article{vollgraf2006quadratic,
  author = {R. Vollgraf and K. Obermayer},
  journal = {IEEE Transactions on Signal Processing},
  number = {9},
  pages = {3270--3278},
  title = {Quadratic optimization for simultaneous matrix diagonalization},
  volume = {54},
  year = {2006},
}
@inproceedings{afsari2006simple,
  author = {B. Afsari},
  booktitle = {Independent Component Analysis and Blind Signal Separation},
  pages = {1--7},
  title = {Simple {LU} and {QR} based non-orthogonal matrix joint diagonalization},
  year = {2006},
}
@inproceedings{guo2010approximate,
  author = {X. Guo and S. Zhu and S. Miron and D. Brie},
  booktitle = {Acoustics Speech and Signal Processing (ICASSP), 2010 IEEE International Conference on},
  pages = {3774--3777},
  title = {Approximate joint diagonalization by nonorthogonal nonparametric jacobi transformations},
  year = {2010},
}
@techreport{anandkumar2014guaranteed,
  author = {A. Anandkumar and R. Ge and M. Janzamin},
  institution = {arXiv},
  title = {Guaranteed Non-Orthogonal Tensor Decomposition via Alternating Rank-1 Updates},
  year = {2014},
}
@article{candes2006near,
  author = {E. J. Candes and T. Tao},
  journal = {IEEE Transactions on Information Theory},
  number = {12},
  pages = {5406--5425},
  title = {Near-optimal signal recovery from random projections: Universal encoding strategies?},
  volume = {52},
  year = {2006},
}
@article{haupt2006signal,
  author = {J. Haupt and R. Nowak},
  journal = {IEEE Transactions on Information Theory},
  number = {9},
  pages = {4036--4048},
  title = {Signal reconstruction from noisy random projections},
  volume = {52},
  year = {2006},
}
@inproceedings{sarlos2006improved,
  author = {T. Sarlos},
  booktitle = {Foundations of Computer Science (FOCS)},
  pages = {143--152},
  title = {Improved approximation algorithms for large matrices via random projections},
  year = {2006},
}
@inproceedings{kleinberg1997two,
  author = {J. M. Kleinberg},
  booktitle = {Symposium on Theory of Computing (STOC)},
  pages = {599--608},
  title = {Two algorithms for nearest-neighbor search in high dimensions},
  year = {1997},
}
@article{delathauwer2001independent,
  author = {L. D. Lathauwer and B. D. Moor and J. Vandewalle},
  journal = {Signal Processing, IEEE Transactions on},
  number = {10},
  pages = {2262--2271},
  title = {Independent component analysis and (simultaneous) third-order tensor diagonalization},
  volume = {49},
  year = {2001},
}
@inproceedings{sutskever2009modelling,
  author = {I. Sutskever and R. Salakhutdinov and J. B. Tenenbaum},
  booktitle = {Advances in Neural Information Processing Systems (NIPS)},
  pages = {1821--1828},
  title = {Modelling Relational Data using {B}ayesian Clustered Tensor Factorization.},
  year = {2009},
}
@inproceedings{nickel2011three,
  author = {M. Nickel and V. Tresp and H. Kriegel},
  booktitle = {International Conference on Machine Learning (ICML)},
  pages = {809--816},
  title = {A three-way model for collective learning on multi-relational data},
  year = {2011},
}
@inproceedings{anandkumar2013community,
  author = {A. Anandkumar and R. Ge and D. Hsu and S. Kakade},
  booktitle = {Conference on Learning Theory (COLT)},
  pages = {867--881},
  title = {A Tensor Spectral Approach to Learning Mixed Membership Community Models},
  year = {2013},
}
@article{boykov2004mincut,
  author = {Y. Boykov and V. Kolmogorov},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {1124--1137},
  title = {An Experimental Comparison of Min-Cut/Max-Flow Algorithms for Energy Minimization in Vision},
  volume = {26},
  year = {2004},
}
@article{boykov2004what,
  author = {V. Kolmogorov and R. Zabih},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {147--159},
  title = {What Energy Functions Can Be Minimized via Graph Cuts?},
  volume = {26},
  year = {2004},
}
@article{boykov2001fast,
  author = {Y. Boykov and O. Veksler and R. Zabih},
  journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (PAMI)},
  pages = {1222--1239},
  title = {Fast Approximate Energy Minimization via Graph Cuts?},
  volume = {23},
  year = {2001},
}
@article{schoenbery1942positive,
  author = {I. J. Schoenberg},
  journal = {Duke Mathematical Journal},
  pages = {96--108},
  title = {Positive definite functions on spheres},
  volume = {9},
  year = {1942},
}
@article{jordan99variational,
  author = {M. I. Jordan and Z. Ghahramani and T. S. Jaakkola and L. K. Saul},
  journal = {Machine Learning},
  pages = {183--233},
  title = {An Introduction to Variational Methods for Graphical Models},
  volume = {37},
  year = {1999},
}
@inproceedings{wei2009non,
  author = {W. Lan and F. D. J and C. Leland and W. H-SP},
  booktitle = {IEEE International Electron Devices Meeting},
  title = {A non-iterative compact model for carbon nanotube FETs incorporating source exhaustion effects},
  year = {2009},
}
@article{natori1998scaling,
  author = {N. Kenji and S. Nobuyuki},
  journal = {Journal of applied physics},
  number = {10},
  pages = {5019--5024},
  title = {Scaling limit of digital circuits due to thermal noise},
  volume = {83},
  year = {1998},
}
@book{weste2010cmos,
  author = {W. Neil and H. David},
  publisher = {Addison-Wesley Publishing Company},
  title = {{CMOS} {VLSI} design: a circuits and systems perspective},
  year = {2010},
}
@article{brachat2010symmetric,
  author = {J. Brachat and P. Comon and B. Mourrain and E. Tsigaridas},
  journal = {Linear Algebra and its Applications},
  number = {11},
  pages = {1851--1872},
  title = {Symmetric tensor decomposition},
  volume = {433},
  year = {2010},
}
@article{comon2009tensor,
  author = {P. Comon and X. Luciani and A. L. D. Almeida},
  journal = {Journal of Chemometrics},
  number = {7},
  pages = {393--405},
  title = {Tensor decompositions, alternating least squares and other tales},
  volume = {23},
  year = {2009},
}
@article{shalev2011online,
  title={Online learning and online convex optimization},
  author={Shalev-Shwartz, Shai},
  journal={Foundations and Trends in Machine Learning},
  volume={4},
  number={2},
  pages={107--194},
  year={2011}
}
@inproceedings{mcmahan2012no,
  title={No-regret algorithms for unconstrained online convex optimization},
  author={Mcmahan, Brendan and Streeter, Matthew},
  booktitle={Advances in neural information processing systems},
  pages={2402--2410},
  year={2012}
}
@article{zeiler2012adadelta,
  title={ADADELTA: An adaptive learning rate method},
  author={Zeiler, Matthew D},
  journal={arXiv preprint arXiv:1212.5701},
  year={2012}
}
@article{schaul2012no,
  title={No more pesky learning rates},
  author={Schaul, Tom and Zhang, Sixin and LeCun, Yann},
  journal={arXiv preprint arXiv:1206.1106},
  year={2012}
}
@article{orabona2014generalized,
  title={A generalized online mirror descent with applications to classification and regression},
  author={Orabona, Francesco and Crammer, Koby and Cesa-Bianchi, Nicolo},
  journal={Machine Learning},
  volume={99},
  number={3},
  pages={411--435},
  year={2014},
  publisher={Springer}
}
@inproceedings{zinkevich2003online,
  title={Online Convex Programming and Generalized Infinitesimal Gradient Ascent},
  author={Zinkevich, Martin},
  booktitle={Proceedings of the 20th International Conference on Machine Learning (ICML-03)},
  pages={928--936},
  year={2003}
}
@inproceedings{abernethy2008optimal,
  title={Optimal strategies and minimax lower bounds for online convex games},
  author={Abernethy, Jacob and Bartlett, Peter L and Rakhlin, Alexander and Tewari, Ambuj},
  booktitle={Proceedings of the nineteenth annual conference on computational learning theory},
  year={2008},
  pages={415--424},
}
@article{chang2011libsvm,
  title={LIBSVM: A library for support vector machines},
  author={Chang, Chih-Chung and Lin, Chih-Jen},
  journal={ACM Transactions on Intelligent Systems and Technology (TIST)},
  volume={2},
  number={3},
  pages={27},
  year={2011},
  publisher={ACM}
}
@inproceedings{guyon2004result,
  title={Result analysis of the nips 2003 feature selection challenge},
  author={Guyon, Isabelle and Gunn, Steve and Ben-Hur, Asa and Dror, Gideon},
  booktitle={Advances in Neural Information Processing Systems},
  pages={545--552},
  year={2004}
}

@inproceedings{chang2001ijcnn,
  title={IJCNN 2001 challenge: Generalization ability and text decoding},
  author={Chang, Chih-chung and Lin, Chih-Jen},
  booktitle={In Proceedings of IJCNN. IEEE},
  year={2001},
  organization={Citeseer}
}
@article{lecun1998gradient,
  title={Gradient-based learning applied to document recognition},
  author={LeCun, Yann and Bottou, L{\'e}on and Bengio, Yoshua and Haffner, Patrick},
  journal={Proceedings of the IEEE},
  volume={86},
  number={11},
  pages={2278--2324},
  year={1998},
  publisher={IEEE}
}
@article{lewis2004rcv1,
  title={Rcv1: A new benchmark collection for text categorization research},
  author={Lewis, David D and Yang, Yiming and Rose, Tony G and Li, Fan},
  journal={The Journal of Machine Learning Research},
  volume={5},
  pages={361--397},
  year={2004},
  publisher={JMLR. org}
}
@article{duarte2004vehicle,
  title={Vehicle classification in distributed sensor networks},
  author={Duarte, Marco F and Hu, Yu Hen},
  journal={Journal of Parallel and Distributed Computing},
  volume={64},
  number={7},
  pages={826--838},
  year={2004},
  publisher={Elsevier}
}
@misc{Lichman:2013 ,
author = "M. Lichman",
year = "2013",
title = "{UCI} Machine Learning Repository",
url = "http://archive.ics.uci.edu/ml",
institution = "University of California, Irvine, School of Information and Computer Sciences" 
} 
@inproceedings{kogan2009predicting,
  title={Predicting risk from financial reports with regression},
  author={Kogan, Shimon and Levin, Dimitry and Routledge, Bryan R and Sagi, Jacob S and Smith, Noah A},
  booktitle={Proceedings of Human Language Technologies: The 2009 Annual Conference of the North American Chapter of the Association for Computational Linguistics},
  pages={272--280},
  year={2009},
  organization={Association for Computational Linguistics}
}
@inproceedings{mcmahan2013minimax,
  title={Minimax optimal algorithms for unconstrained linear optimization},
  author={McMahan, Brendan and Abernethy, Jacob},
  booktitle={Advances in Neural Information Processing Systems},
  pages={2724--2732},
  year={2013}
}
@article{auer2002adaptive,
  title={Adaptive and self-confident on-line learning algorithms},
  author={Auer, Peter and Cesa-Bianchi, Nicolo and Gentile, Claudio},
  journal={Journal of Computer and System Sciences},
  volume={64},
  number={1},
  pages={48--75},
  year={2002},
  publisher={Elsevier}
}

@inproceedings{mcmahan2010adaptive,
title = {Adaptive Bound Optimization for Online Convex Optimization},
author  = {H. Brendan McMahan and Matthew Streeter},
year  = 2010,
pages     = {244--256},
booktitle = {Proceedings of the 23rd Annual Conference on Learning Theory (COLT)}
}

@article{vershynin2010introduction,
  title={Introduction to the non-asymptotic analysis of random matrices},
  author={Vershynin, Roman},
  journal={arXiv preprint arXiv:1011.3027},
  year={2010}
}

@article{orabona2016scale,
  title={Scale-Free Online Learning},
  author={Orabona, Francesco and P{\'a}l, D{\'a}vid},
  journal={arXiv preprint arXiv:1601.01974},
  year={2016}
}
@inproceedings{orabona2014simultaneous,
  title={Simultaneous model selection and optimization through parameter-free stochastic learning},
  author={Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1116--1124},
  year={2014}
}


@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}
@inproceedings{orabona2013dimension,
  title={Dimension-free exponentiated gradient},
  author={Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1806--1814},
  year={2013}
}

@inproceedings{littlestone2014line,
  title={From on-line to batch learning},
  author={Littlestone, Nick},
  booktitle={Proceedings of the second annual workshop on Computational learning theory},
  pages={269--284},
  year={2014}
}

@article{cesa2004generalization,
  title={On the generalization ability of on-line learning algorithms},
  author={Cesa-Bianchi, Nicolo and Conconi, Alex and Gentile, Claudio},
  journal={Information Theory, IEEE Transactions on},
  volume={50},
  number={9},
  pages={2050--2057},
  year={2004},
  publisher={IEEE}
}

@article{mcmahan2014survey,
title = {A Survey of Algorithms and Analysis for Adaptive Online Learning},
author  = {H. Brendan McMahan},
journal={arXiv preprint arXiv:1403.3465},
year  = 2014
}

@article{hofmann2008kernel,
  title={Kernel methods in machine learning},
  author={Hofmann, Thomas and Sch{\"o}lkopf, Bernhard and Smola, Alexander J},
  journal={The annals of statistics},
  pages={1171--1220},
  year={2008},
  publisher={JSTOR}
}

@misc{krizhevsky2009learning,
  title={Learning multiple layers of features from tiny images},
  author={Krizhevsky, Alex and Hinton, Geoffrey},
  year={2009},
  publisher={Citeseer}
}

@inproceedings{orabona2016open,
  title={Open Problem: Parameter-Free and Scale-Free Online Algorithms},
  author={Orabona, Francesco and P{\'a}l, D{\'a}vid},
  booktitle={Conference on Learning Theory},
  year={2016}
}

@incollection{orabona2016coin,
title = {Coin Betting and Parameter-Free Online Learning},
author = {Orabona, Francesco and P{\'a}l, D{\'a}vid},
booktitle = {Advances in Neural Information Processing Systems 29},
editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett},
pages = {577--585},
year = {2016},
publisher = {Curran Associates, Inc.},
}

@incollection{cutkosky2016online,
title = {Online Convex Optimization with Unconstrained Domains and Losses},
author = {Cutkosky, Ashok and Boahen, Kwabena A},
booktitle = {Advances in Neural Information Processing Systems 29},
editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett},
pages = {748--756},
year = {2016},
publisher = {Curran Associates, Inc.},
}
@inproceedings{cutkosky2017online,
  title={Online Learning Without Prior Information},
  author={Cutkosky, Ashok and Boahen, Kwabena},
  booktitle={Conference on Learning Theory},
  pages={643--677},
  year={2017}
}

@inproceedings{cutkosky2018distributed,
  title={Distributed stochastic optimization via adaptive SGD},
  author={Cutkosky, Ashok and Busa-Fekete, R{\'o}bert},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1910--1919},
  year={2018}
}

@inproceedings{cutkosky2017stochastic,
  title={Stochastic and adversarial online learning without hyperparameters},
  author={Cutkosky, Ashok and Boahen, Kwabena A},
  booktitle={Advances in Neural Information Processing Systems},
  pages={5059--5067},
  year={2017}
}
@inproceedings{cutkosky2019matrix,
  title={Matrix-Free Preconditioning in Online Learning},
  author={Cutkosky, Ashok and Sarlos, Tamas},
  booktitle={International Conference on Machine Learning},
  pages={1455--1464},
  year={2019}
}

@inproceedings{cutkosky2020parameter,
  title={Parameter-free, Dynamic, and Strongly-Adaptive Online Learning},
  author={Cutkosky, Ashok},
  booktitle={International Conference on Machine Learning},
  volume={2},
  year={2020}
}

@inproceedings{cutkosky2019artificial,
  title = 	 {Artificial Constraints and Hints for Unbounded Online Learning},
  author = 	 {Cutkosky, Ashok},
  booktitle = 	 {Proceedings of the Thirty-Second Conference on Learning Theory},
  pages = 	 {874--894},
  year = 	 {2019},
}

@inproceedings{cutkosky2019anytime,
  title={Anytime Online-to-Batch, Optimism and Acceleration},
  author={Cutkosky, Ashok},
  booktitle={International Conference on Machine Learning},
  pages={1446--1454},
  year={2019}
}
@inproceedings{zhuang2019surrogate,
  title={Surrogate Losses for Online Learning of Stepsizes in Stochastic Non-Convex Optimization},
  author={Zhuang, Zhenxun and Cutkosky, Ashok and Orabona, Francesco},
  booktitle={International Conference on Machine Learning},
  pages={7664--7672},
  year={2019}
}

@inproceedings{cutkosky2019momentum,
  title={Momentum-Based Variance Reduction in Non-convex SGD},
  author={Cutkosky, Ashok and Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019},
  pages={15210--15219}
}

@inproceedings{cutkosky2019kernel,
  title={Kernel Truncated Randomized Ridge Regression: Optimal Rates and Low Noise Acceleration},
  author={Jun, Kwang-Sung, amd Cutkosky, Ashok and Orabona, Francesco},
  booktitle={Advances in Neural Information Processing Systems},
  year={2019}
}

@InProceedings{cutkosky2019combining,
  title = 	 {Combining Online Learning Guarantees},
  author = 	 {Cutkosky, Ashok},
  booktitle = 	 {Proceedings of the Thirty-Second Conference on Learning Theory},
  pages = 	 {895--913},
  year = 	 {2019}
}

@incollection{vanervan2016metagrad,
title = {MetaGrad: Multiple Learning Rates in Online Learning},
author = {van Erven, Tim and Koolen, Wouter M},
booktitle = {Advances in Neural Information Processing Systems 29},
editor = {D. D. Lee and M. Sugiyama and U. V. Luxburg and I. Guyon and R. Garnett},
pages = {3666--3674},
year = {2016},
publisher = {Curran Associates, Inc.},
}

@inproceedings{cutkosky2015bloom,
  title={Bloom Features},
  author={Cutkosky, Ashok and Boahen, Kwabena},
  booktitle={2015 International Conference on Computational Science and Computational Intelligence (CSCI)},
  pages={547--552},
  year={2015},
  organization={IEEE}
}


@inproceedings{mcmahan2014unconstrained,
  title={Unconstrained Online Linear Learning in Hilbert Spaces: Minimax Algorithms and Normal Approximations.},
  author={McMahan, H Brendan and Orabona, Francesco},
  booktitle={COLT},
  pages={1020--1039},
  year={2014}
}
@inproceedings{luo15achieving,
  author = {Luo, Haipeng and Schapire, Robert},
  booktitle = {Conference on Learning Theory (COLT)},
  pages = {1286-–1304},
  title = {Achieving All with No Parameters: AdaNormalHedge},
  year = {2015},
}

@inproceedings{srebro2010smoothness,
  title={Smoothness, low noise and fast rates},
  author={Srebro, Nathan and Sridharan, Karthik and Tewari, Ambuj},
  booktitle={Advances in neural information processing systems},
  pages={2199--2207},
  year={2010}
}

@inproceedings{ross2013normalized,
  title={Normalized Online Learning},
  author={Ross, Stephane, Mineiro, Paul, and Langford, John},
  booktitle={Proceedings of the 29th Conference on Uncertainty in Artificial Intelligence (UAI)},
  year={2013}
} 

@incollection{foster2015adaptive,
title = {Adaptive Online Learning},
author = {Foster, Dylan J and Rakhlin, Alexander and Sridharan, Karthik},
booktitle = {Advances in Neural Information Processing Systems 28},
editor = {C. Cortes and N. D. Lawrence and D. D. Lee and M. Sugiyama and R. Garnett},
pages = {3375--3383},
year = {2015},
publisher = {Curran Associates, Inc.},
}


@inproceedings{foster2017parameter,
  title={Parameter-free online learning via model selection},
  author={Foster, Dylan J and Kale, Satyen and Mohri, Mehryar and Sridharan, Karthik},
  booktitle={Advances in Neural Information Processing Systems},
  pages={6020--6030},
  year={2017}
}

@inproceedings{cutkosky2018black,
  title={Black-Box Reductions for Parameter-free Online Learning in Banach Spaces},
  author={Cutkosky, Ashok and Orabona, Francesco},
  booktitle={Conference On Learning Theory},
  pages={1493--1529},
  year={2018}
}

@inproceedings{mohri2016accelerating,
  title={Accelerating online convex optimization via adaptive prediction},
  author={Mohri, Mehryar and Yang, Scott},
  booktitle={Artificial Intelligence and Statistics},
  pages={848--856},
  year={2016}
}

@inproceedings{joulani2017modular,
  author    = {Pooria Joulani and
              Andr{\'{a}}s Gy{\"{o}}rgy and
              Csaba Szepesv{\'{a}}ri},
  title     = {A Modular Analysis of Adaptive (Non-)Convex Optimization: Optimism,
              Composite Objectives, and Variational Bounds},
  booktitle = {International Conference on Algorithmic Learning Theory, {ALT} 2017,
              15-17 October 2017, Kyoto University, Kyoto, Japan},
  pages     = {681--720},
  year      = {2017},
  url       = {http://proceedings.mlr.press/v76/joulani17a.html},
  timestamp = {Mon, 16 Apr 2018 12:15:20 +0200},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{foster2018online,
  author    = {Dylan J. Foster and
              Alexander Rakhlin and
              Karthik Sridharan},
  title     = {Online Learning: Sufficient Statistics and the Burkholder Method},
  booktitle = {Conference on Learning Theory (COLT)},
  year      = {2018}
}

@inproceedings{orabona2017training,
  author    = {Francesco Orabona and
              Tatiana Tommasi},
  title     = {Training Deep Networks without Learning Rates Through Coin Betting},
  booktitle = {Advances in Neural Information Processing Systems 30: Annual Conference
              on Neural Information Processing Systems 2017, 4-9 December 2017,
              Long Beach, CA, {USA}},
  pages     = {2157--2167},
  year      = {2017},
  url       = {http://papers.nips.cc/paper/6811-training-deep-networks-without-learning-rates-through-coin-betting},
  timestamp = {Mon, 27 Nov 2017 12:38:48 +0100},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{rakhlin2013online,
  author    = {Alexander Rakhlin and
              Karthik Sridharan},
  title     = {Online Learning with Predictable Sequences},
  booktitle = {Conference on Learning Theory (COLT)},
  pages     = {993--1019},
  year      = {2013}
}

@article{hazan2010extracting,
  title={Extracting certainty from uncertainty: Regret bounded by variation in costs},
  author={Hazan, Elad and Kale, Satyen},
  journal={Machine learning},
  volume={80},
  number={2-3},
  pages={165--188},
  year={2010},
  publisher={Springer}
}

@inproceedings{chiang2012online,
  title={Online optimization with gradual variations},
  author={Chiang, Chao-Kai and Yang, Tianbao and Lee, Chia-Jung and Mahdavi, Mehrdad and Lu, Chi-Jen and Jin, Rong and Zhu, Shenghuo},
  booktitle={Conference on Learning Theory},
  pages={6--1},
  year={2012}
}

@article{rakhlin2015equivalence,
  title={On equivalence of martingale tail bounds and deterministic regret inequalities},
  author={Rakhlin, Alexander and Sridharan, Karthik},
  journal={arXiv preprint arXiv:1510.03925},
  year={2015}
}

@article{maurer2009empirical,
  title={Empirical Bernstein bounds and sample variance penalization},
  author={Maurer, Andreas and Pontil, Massimiliano},
  journal={arXiv preprint arXiv:0907.3740},
  year={2009}
}

@inproceedings{hazan2008adaptive,
  title={Adaptive online gradient descent},
  author={Hazan, Elad and Rakhlin, Alexander and Bartlett, Peter L},
  booktitle={Advances in Neural Information Processing Systems},
  pages={65--72},
  year={2008}
}

@inproceedings{ge2019open,
  title={Open Problem: Do Good Algorithms Necessarily Query Bad Points?},
  author={Ge, Rong and Jain, Prateek and Kakade, Sham M and Kidambi, Rahul and Nagaraj, Dheeraj M and Netrapalli, Praneeth},
  booktitle={Conference on Learning Theory},
  pages={3190--3193},
  year={2019}
}

@article{arjevani2019lower,
  title={Lower bounds for non-convex stochastic optimization},
  author={Arjevani, Yossi and Carmon, Yair and Duchi, John C and Foster, Dylan J and Srebro, Nathan and Woodworth, Blake},
  journal={arXiv preprint arXiv:1912.02365},
  year={2019}
}
@inproceedings{arjevani2020second,
  title={Second-Order Information in Non-Convex Stochastic Optimization: Power and Limitations},
  author={Arjevani, Yossi and Carmon, Yair and Duchi, John C and Foster, Dylan J and Sekhari, Ayush and Sridharan, Karthik},
  booktitle={Conference on Learning Theory},
  pages={242--299},
  year={2020}
}

@inproceedings{fang2018spider,
  title={Spider: Near-optimal non-convex optimization via stochastic path-integrated differential estimator},
  author={Fang, Cong and Li, Chris Junchi and Lin, Zhouchen and Zhang, Tong},
  booktitle={Advances in Neural Information Processing Systems},
  pages={689--699},
  year={2018}
}

@inproceedings{fang2019sharp,
  title={Sharp Analysis for Nonconvex SGD Escaping from Saddle Points},
  author={Fang, Cong and Lin, Zhouchen and Zhang, Tong},
  booktitle={Conference on Learning Theory},
  pages={1192--1234},
  year={2019}
}
@inproceedings{cutkosky2020momentum,
  title={Momentum Improves Normalized SGD},
  author={Cutkosky, Ashok and Mehta, Harsh},
  booktitle={International Conference on Machine Learning},
  year={2020}
}

@inproceedings{johnson2013accelerating,
  title={Accelerating stochastic gradient descent using predictive variance reduction},
  author={Johnson, Rie and Zhang, Tong},
  booktitle={Advances in neural information processing systems},
  pages={315--323},
  year={2013}
}

@article{tran2019hybrid,
  title={A Hybrid Stochastic Optimization Framework for Stochastic Composite Nonconvex Optimization},
  author={Tran-Dinh, Quoc and Pham, Nhan H and Phan, Dzung T and Nguyen, Lam M},
  journal={arXiv preprint arXiv:1907.03793},
  year={2019}
}

@inproceedings{allen2018natasha,
  title={Natasha 2: Faster non-convex optimization than sgd},
  author={Allen-Zhu, Zeyuan},
  booktitle={Advances in neural information processing systems},
  pages={2675--2686},
  year={2018}
}



@inproceedings{
reddi2018on,
title={On the Convergence of Adam and Beyond},
author={Sashank J. Reddi and Satyen Kale and Sanjiv Kumar},
booktitle={International Conference on Learning Representations},
year={2018}
}
@inproceedings{imagenet_cvpr09,
AUTHOR = {Deng, J. and Dong, W. and Socher, R. and Li, L.-J. and Li, K. and Fei-Fei, L.},
TITLE = {{ImageNet: A Large-Scale Hierarchical Image Database}},
BOOKTITLE = {CVPR09},
YEAR = {2009},
BIBSOURCE = "http://www.image-net.org/papers/imagenet_cvpr09.bib"
}
@inproceedings{devlin-etal-2019-bert,
    title = "{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding",
    author = "Devlin, Jacob  and
      Chang, Ming-Wei  and
      Lee, Kenton  and
      Toutanova, Kristina",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/N19-1423",
    doi = "10.18653/v1/N19-1423",
    pages = "4171--4186",
    abstract = "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5 (7.7 point absolute improvement), MultiNLI accuracy to 86.7{\%} (4.6{\%} absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).",
}
@article{Raffel2019ExploringTL,
  title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
  author={Colin Raffel and Noam Shazeer and Adam Kaleo Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
  journal={ArXiv},
  year={2019},
  volume={abs/1910.10683}
}
@misc{shazeer2017outrageously,
    title={Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer},
    author={Noam Shazeer and Azalia Mirhoseini and Krzysztof Maziarz and Andy Davis and Quoc Le and Geoffrey Hinton and Jeff Dean},
    year={2017},
    eprint={1701.06538},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

@article{radford2019language,
  title={Language Models are Unsupervised Multitask Learners},
  author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
  year={2019},
  journal={OpenAI ech report}
}


@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom B and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel M. and Wu, Jeffrey Winter, Clemens and Hesse, Christopher and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}
@MISC{li2020openai,
author = {Li, Chuan},
title = {OpenAI's GPT-3 Language Model: A Technical Overview},
year = {2020},
howpublished={\url{https://lambdalabs.com/blog/demystifying-gpt-3/}}
}

@article{kelley2018sequential,
  title={Sequential regulatory activity prediction across chromosomes with convolutional neural networks},
  author={Kelley, David R and Reshef, Yakir A and Bileschi, Maxwell and Belanger, David and McLean, Cory Y and Snoek, Jasper},
  journal={Genome research},
  volume={28},
  number={5},
  pages={739--750},
  year={2018},
  publisher={Cold Spring Harbor Lab}
}


@article{halstead2020comparative,
  title={A comparative analysis of chromatin accessibility in cattle, pig, and mouse tissues},
  author={Halstead, Michelle M and Kern, Colin and Saelao, Perot and Wang, Ying and Chanthavixay, Ganrea and Medrano, Juan F and Van Eenennaam, Alison L and Korf, Ian and Tuggle, Christopher K and Ernst, Catherine W and others},
  journal={BMC genomics},
  volume={21},
  number={1},
  pages={1--16},
  year={2020},
  publisher={BioMed Central}
}

@article{kern2018genome,
  title={Genome-wide identification of tissue-specific long non-coding RNA in three farm animal species},
  author={Kern, Colin and Wang, Ying and Chitwood, James and Korf, Ian and Delany, Mary and Cheng, Hans and Medrano, Juan F and Van Eenennaam, Alison L and Ernst, Catherine and Ross, Pablo and others},
  journal={BMC genomics},
  volume={19},
  number={1},
  pages={1--14},
  year={2018},
  publisher={Springer}
}

@article{halstead2020systematic,
  title={Systematic alteration of ATAC-seq for profiling open chromatin in cryopreserved nuclei preparations from livestock tissues},
  author={Halstead, MM and Kern, C and Saelao, P and Chanthavixay, G and Wang, Y and Delany, ME and Zhou, H and Ross, PJ},
  journal={Scientific reports},
  volume={10},
  number={1},
  pages={1--12},
  year={2020},
  publisher={Nature Publishing Group}
}



@MISC{apple21privacy,
author = {Apple Inc.},
title = {Apple Privacy Website},
year = {2021},
howpublished={\url{https://www.apple.com/privacy/control/}}
}

@MISC{google21privacy,
author = {Google Differential Privacy Team},
title = {Differential Privacy Github Repository},
year = {2021},
howpublished={\url{https://github.com/google/differential-privacy}}
}



@article{li2017hyperband,
  title={Hyperband: A novel bandit-based approach to hyperparameter optimization},
  author={Li, Lisha and Jamieson, Kevin and DeSalvo, Giulia and Rostamizadeh, Afshin and Talwalkar, Ameet},
  journal={The Journal of Machine Learning Research},
  volume={18},
  number={1},
  pages={6765--6816},
  year={2017},
  publisher={JMLR. org}
}
@inproceedings{golovin2017google,
  title={Google vizier: A service for black-box optimization},
  author={Golovin, Daniel and Solnik, Benjamin and Moitra, Subhodeep and Kochanski, Greg and Karro, John and Sculley, D},
  booktitle={Proceedings of the 23rd ACM SIGKDD international conference on knowledge discovery and data mining},
  pages={1487--1495},
  year={2017}
}


@article{falkner2018bohb,
  title={BOHB: Robust and efficient hyperparameter optimization at scale},
  author={Falkner, Stefan and Klein, Aaron and Hutter, Frank},
  journal={arXiv preprint arXiv:1807.01774},
  year={2018}
}



@article{hazan2017hyperparameter,
  title={Hyperparameter optimization: A spectral approach},
  author={Hazan, Elad and Klivans, Adam and Yuan, Yang},
  journal={arXiv preprint arXiv:1706.00764},
  year={2017}
}

@article{chen2020better,
  title={Better Parameter-free Stochastic Optimization with ODE Updates for Coin-Betting},
  author={Chen, Keyi and Langford, John and Orabona, Francesco},
  journal={arXiv preprint arXiv:2006.07507},
  year={2020}
}

@inproceedings{defazio2019ineffectiveness,
  title={On the ineffectiveness of variance reduced optimization for deep learning},
  author={Defazio, Aaron and Bottou, L{\'e}on},
  booktitle={Advances in Neural Information Processing Systems},
  pages={1755--1765},
  year={2019}
}

@inproceedings{tripuraneni2018stochastic,
  title={Stochastic cubic regularization for fast nonconvex optimization},
  author={Tripuraneni, Nilesh and Stern, Mitchell and Jin, Chi and Regier, Jeffrey and Jordan, Michael I},
  booktitle={Advances in neural information processing systems},
  pages={2899--2908},
  year={2018}
}

@inproceedings{van2019user,
  title={User-specified local differential privacy in unconstrained adaptive online learning},
  author={van der Hoeven, Dirk},
  booktitle={Advances in Neural Information Processing Systems},
  pages={14103--14112},
  year={2019}
}

@article{mhammedi2020lipschitz,
  title={Lipschitz and Comparator-Norm Adaptivity in Online Learning},
  author={Mhammedi, Zakaria and Koolen, Wouter M},
  journal={Conference on Learning Theory},
  year={2020},
  pages={2858--2887},
}

@inproceedings{allen2017linear,
  title={Linear Coupling: An Ultimate Unification of Gradient and Mirror Descent},
  author={Allen-Zhu, Zeyuan and Orecchia, Lorenzo},
  booktitle={8th Innovations in Theoretical Computer Science Conference (ITCS 2017)},
  year={2017},
  organization={Schloss Dagstuhl-Leibniz-Zentrum fuer Informatik}
}

@article{joulani2020simpler,
  title={A Simpler Approach to Accelerated Stochastic Optimization: Iterative Averaging Meets Optimism},
  author={Joulani, Pooria and Raj, Anant and Gy{\"o}rgy, Andr{\'a}s and Szepesv{\'a}ri, Csaba},
  year={2020},
  pages ={4984--4993},
  journal={Internation Conference on Machine Learning}
}
@inproceedings{kempka2019adaptive,
  title={Adaptive Scale-Invariant Online Algorithms for Learning Linear Models},
  author={Kempka, Michal and Kotlowski, Wojciech and Warmuth, Manfred K},
  booktitle={International Conference on Machine Learning},
  pages={3321--3330},
  year={2019}
}


@article{dwork2014algorithmic,
  title={The algorithmic foundations of differential privacy.},
  author={Dwork, Cynthia and Roth, Aaron},
  journal={Found. Trends Theor. Comput. Sci.},
  volume={9},
  number={3-4},
  pages={211--407},
  year={2014}
}


@inproceedings{bassily2019private,
  title={Private stochastic convex optimization with optimal rates},
  author={Bassily, Raef and Feldman, Vitaly and Talwar, Kunal and Thakurta, Abhradeep},
  booktitle={Proceedings of the 33rd International Conference on Neural Information Processing Systems},
  pages={11282--11291},
  year={2019}
}

@inproceedings{feldman2020private,
  title={Private stochastic convex optimization: optimal rates in linear time},
  author={Feldman, Vitaly and Koren, Tomer and Talwar, Kunal},
  booktitle={Proceedings of the 52nd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={439--449},
  year={2020}
}

@inproceedings{hardt2016train,
  title={Train faster, generalize better: Stability of stochastic gradient descent},
  author={Hardt, Moritz and Recht, Ben and Singer, Yoram},
  booktitle={International Conference on Machine Learning},
  pages={1225--1234},
  year={2016},
  organization={PMLR}
}
@inproceedings{feldman2018privacy,
  title={Privacy amplification by iteration},
  author={Feldman, Vitaly and Mironov, Ilya and Talwar, Kunal and Thakurta, Abhradeep},
  booktitle={2018 IEEE 59th Annual Symposium on Foundations of Computer Science (FOCS)},
  pages={521--532},
  year={2018},
  organization={IEEE}
}

@article{zhou2020private,
  title={Private Stochastic Non-Convex Optimization: Adaptive Algorithms and Tighter Generalization Bounds},
  author={Zhou, Yingxue and Chen, Xiangyi and Hong, Mingyi and Wu, Zhiwei Steven and Banerjee, Arindam},
  journal={arXiv preprint arXiv:2006.13501},
  year={2020}
}


@article{guzman2021differentially,
  title={Differentially Private Stochastic Optimization: New Results in Convex and Non-Convex Settings},
  author={Bassily, Raef and Guzm{\'a}n, Crist{\'o}bal and Menart, Michael},
  journal={arXiv preprint arXiv:2107.05585},
  year={2021}
}

@inproceedings{tramer2020differentially,
  title={Differentially Private Learning Needs Better Features (or Much More Data)},
  author={Tramer, Florian and Boneh, Dan},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@article{anil2021large,
  title={Large-Scale Differentially Private BERT},
  author={Anil, Rohan and Ghazi, Badih and Gupta, Vineet and Kumar, Ravi and Manurangsi, Pasin},
  journal={arXiv preprint arXiv:2108.01624},
  year={2021}
}


@article{tran2021correcting,
  title={Correcting Momentum with Second-order Information},
  author={Tran, Hoang and Cutkosky, Ashok},
  journal={arXiv preprint arXiv:2103.03265},
  year={2021}
}


@inproceedings{lecuyer2019certified,
  title={Certified robustness to adversarial examples with differential privacy},
  author={Lecuyer, Mathias and Atlidakis, Vaggelis and Geambasu, Roxana and Hsu, Daniel and Jana, Suman},
  booktitle={2019 IEEE Symposium on Security and Privacy (SP)},
  pages={656--672},
  year={2019},
  organization={IEEE}
}


@inproceedings{abadi2016deep,
  title={Deep learning with differential privacy},
  author={Abadi, Martin and Chu, Andy and Goodfellow, Ian and McMahan, H Brendan and Mironov, Ilya and Talwar, Kunal and Zhang, Li},
  booktitle={Proceedings of the 2016 ACM SIGSAC conference on computer and communications security},
  pages={308--318},
  year={2016}
}


@article{wang2019efficient,
  title={Efficient Privacy-Preserving Stochastic Nonconvex Optimization},
  author={Wang, Lingxiao and Jayaraman, Bargav and Evans, David and Gu, Quanquan},
  journal={arXiv preprint arXiv:1910.13659},
  year={2019}
}

@inproceedings{wang2019differentially,
  title={Differentially private empirical risk minimization with non-convex loss functions},
  author={Wang, Di and Chen, Changyou and Xu, Jinhui},
  booktitle={International Conference on Machine Learning},
  pages={6526--6535},
  year={2019},
  organization={PMLR}
}

@inproceedings{wang2017differentially,
  title={Differentially private empirical risk minimization revisited: faster and more general},
  author={Wang, Di and Ye, Minwei and Xu, Jinhui},
  booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
  pages={2719--2728},
  year={2017}
}

@inproceedings{zhang2017efficient,
  title={Efficient private ERM for smooth objectives},
  author={Zhang, Jiaqi and Zheng, Kai and Mou, Wenlong and Wang, Liwei},
  booktitle={Proceedings of the 26th International Joint Conference on Artificial Intelligence},
  pages={3922--3928},
  year={2017}
}

@inproceedings{mcmahan2018learning,
  title={Learning Differentially Private Recurrent Language Models},
  author={McMahan, H Brendan and Ramage, Daniel and Talwar, Kunal and Zhang, Li},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{li2020high,
  title={A High Probability Analysis of Adaptive SGD with Momentum},
  author={Li, Xiaoyu and Orabona, Francesco},
  booktitle={Workshop on Beyond First Order Methods in ML Systems at ICML'20},
  year={2020}
}
@inproceedings{cutkosky2021high,
  title={High-probability Bounds for Non-Convex Stochastic Optimization with Heavy Tails},
  author={Cutkosky, Ashok and Mehta, Harsh},
  booktitle={Proceedings of the 35st International Conference on Neural Information Processing Systems},
  year={2021},
  pages={4883--4895}
}

@article{ghadimi2013stochastic,
  title={Stochastic first-and zeroth-order methods for nonconvex stochastic programming},
  author={Ghadimi, Saeed and Lan, Guanghui},
  journal={SIAM Journal on Optimization},
  volume={23},
  number={4},
  pages={2341--2368},
  year={2013},
  publisher={SIAM}
}


@article{chan2011private,
  title={Private and continual release of statistics},
  author={Chan, T-H Hubert and Shi, Elaine and Song, Dawn},
  journal={ACM Transactions on Information and System Security (TISSEC)},
  volume={14},
  number={3},
  pages={1--24},
  year={2011},
  publisher={ACM New York, NY, USA}
}

@inproceedings{dwork2010differential,
  title={Differential privacy under continual observation},
  author={Dwork, Cynthia and Naor, Moni and Pitassi, Toniann and Rothblum, Guy N},
  booktitle={Proceedings of the forty-second ACM symposium on Theory of computing},
  pages={715--724},
  year={2010}
}

@inproceedings{dwork2015preserving,
  title={Preserving statistical validity in adaptive data analysis},
  author={Dwork, Cynthia and Feldman, Vitaly and Hardt, Moritz and Pitassi, Toniann and Reingold, Omer and Roth, Aaron Leon},
  booktitle={Proceedings of the forty-seventh annual ACM symposium on Theory of computing},
  pages={117--126},
  year={2015}
}

@inproceedings{jung2021new,
  title={A new analysis of differential privacy’s generalization guarantees},
  author={Jung, Christopher and Ligett, Katrina and Neel, Seth and Roth, Aaron and Sharifi-Malvajerdi, Saeed and Shenfeld, Moshe},
  booktitle={Proceedings of the 53rd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={9--9},
  year={2021}
}


@inproceedings{zhou2018critical,
  title={Critical points of linear neural networks: Analytical forms and landscape properties},
  author={Zhou, Yi and Liang, Yingbin},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{choromanska2015loss,
  title={The loss surfaces of multilayer networks},
  author={Choromanska, Anna and Henaff, Mikael and Mathieu, Michael and Arous, G{\'e}rard Ben and LeCun, Yann},
  booktitle={Artificial intelligence and statistics},
  pages={192--204},
  year={2015},
  organization={PMLR}
}

@inproceedings{kawaguchi2016deep,
  title={Deep learning without poor local minima},
  author={Kawaguchi, Kenji},
  booktitle={Proceedings of the 30th International Conference on Neural Information Processing Systems},
  pages={586--594},
  year={2016}
}

@article{basu2021benchmarking,
  title={Benchmarking Differential Privacy and Federated Learning for BERT Models},
  author={Basu, Priyam and Roy, Tiasa Singha and Naidu, Rakshit and Muftuoglu, Zumrut and Singh, Sahib and Mireshghallah, Fatemehsadat},
  journal={arXiv preprint arXiv:2106.13973},
  year={2021}
}

@inproceedings{zhou2018stochastic,
  title={Stochastic nested variance reduction for nonconvex optimization},
  author={Zhou, Dongruo and Xu, Pan and Gu, Quanquan},
  booktitle={Proceedings of the 32nd International Conference on Neural Information Processing Systems},
  pages={3925--3936},
  year={2018}
}

@article{hazan2019introduction,
  title={Introduction to online convex optimization},
  author={Hazan, Elad},
  journal={arXiv preprint arXiv:1909.05207},
  year={2019}
}

@article{guha2013nearly,
  title={(Nearly) optimal algorithms for private online learning in full-information and bandit settings},
  author={Guha Thakurta, Abhradeep and Smith, Adam},
  journal={Advances in Neural Information Processing Systems},
  volume={26},
  pages={2733--2741},
  year={2013}
}

@inproceedings{jain2012differentially,
  title={Differentially private online learning},
  author={Jain, Prateek and Kothari, Pravesh and Thakurta, Abhradeep},
  booktitle={Conference on Learning Theory},
  pages={24--1},
  year={2012},
  organization={JMLR Workshop and Conference Proceedings}
}

@inproceedings{agarwal2017price,
  title={The price of differential privacy for online learning},
  author={Agarwal, Naman and Singh, Karan},
  booktitle={International Conference on Machine Learning},
  pages={32--40},
  year={2017},
  organization={PMLR}
}


@article{kairouz2021practical,
  title={Practical and private (deep) learning without sampling or shuffling},
  author={Kairouz, Peter and McMahan, Brendan and Song, Shuang and Thakkar, Om and Thakurta, Abhradeep and Xu, Zheng},
  journal={arXiv preprint arXiv:2103.00039},
  year={2021}
}

@inproceedings{jain2014near,
  title={(Near) dimension independent risk bounds for differentially private learning},
  author={Jain, Prateek and Thakurta, Abhradeep Guha},
  booktitle={International Conference on Machine Learning},
  pages={476--484},
  year={2014},
  organization={PMLR}
}

@article{orabona2019modern,
  title={A modern introduction to online learning},
  author={Orabona, Francesco},
  journal={arXiv preprint arXiv:1912.13213},
  year={2019}
}

@article{hazan2007logarithmic,
  title={Logarithmic regret algorithms for online convex optimization},
  author={Hazan, Elad and Agarwal, Amit and Kale, Satyen},
  journal={Machine Learning},
  volume={69},
  number={2-3},
  pages={169--192},
  year={2007},
  publisher={Springer}
}

@inproceedings{zhang2018adaptive,
  title={Adaptive online learning in dynamic environments},
  author={Zhang, Lijun and Lu, Shiyin and Zhou, Zhi-Hua},
  booktitle={Proceedings of the 32nd International Conference on Neural Information Processing Systems},
  pages={1330--1340},
  year={2018}
}

@article{agarwal2009information,
  title={Information-theoretic lower bounds on the oracle complexity of convex optimization},
  author={Agarwal, Alekh and Wainwright, Martin J and Bartlett, Peter and Ravikumar, Pradeep},
  journal={Advances in Neural Information Processing Systems},
  volume={22},
  pages={1--9},
  year={2009}
}

@article{levy2019necessary,
  title={Necessary and Sufficient Geometries for Gradient Methods},
  author={Levy, Daniel and Duchi, John C},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  pages={11495--11505},
  year={2019}
}

@article{kairouz2020dimension,
  title={Dimension independence in unconstrained private erm via adaptive preconditioning},
  author={Kairouz, Peter and Ribero, M{\'o}nica and Rush, Keith and Thakurta, Abhradeep},
  journal={arXiv e-prints},
  pages={arXiv--2008},
  year={2020}
}



@article{li2021large,
  title={Large Language Models Can Be Strong Differentially Private Learners},
  author={Li, Xuechen and Tram{\`e}r, Florian and Liang, Percy and Hashimoto, Tatsunori},
  journal={arXiv preprint arXiv:2110.05679},
  year={2021}
}

@article{yu2021differentially,
  title={Differentially Private Fine-tuning of Language Models},
  author={Yu, Da and Naik, Saurabh and Backurs, Arturs and Gopi, Sivakanth and Inan, Huseyin A and Kamath, Gautam and Kulkarni, Janardhan and Lee, Yin Tat and Manoel, Andre and Wutschitz, Lukas and Yekhanin, Sergey and Zhang, Huishuai},
  journal={arXiv preprint arXiv:2110.06500},
  year={2021}
}

@article{zhang2019fixup,
  title={Fixup initialization: Residual learning without normalization},
  author={Zhang, Hongyi and Dauphin, Yann N and Ma, Tengyu},
  journal={arXiv preprint arXiv:1901.09321},
  year={2019}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@inproceedings{glorot2010understanding,
  title={Understanding the difficulty of training deep feedforward neural networks},
  author={Glorot, Xavier and Bengio, Yoshua},
  booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics},
  pages={249--256},
  year={2010},
  organization={JMLR Workshop and Conference Proceedings}
}

@article{srivastava2014dropout,
  title={Dropout: a simple way to prevent neural networks from overfitting},
  author={Srivastava, Nitish and Hinton, Geoffrey and Krizhevsky, Alex and Sutskever, Ilya and Salakhutdinov, Ruslan},
  journal={The journal of machine learning research},
  volume={15},
  number={1},
  pages={1929--1958},
  year={2014},
  publisher={JMLR. org}
}

@article{ba2016layer,
  title={Layer normalization},
  author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
  journal={arXiv preprint arXiv:1607.06450},
  year={2016}
}

@inproceedings{jorgensen2015conservative,
  title={Conservative or liberal? Personalized differential privacy},
  author={Jorgensen, Zach and Yu, Ting and Cormode, Graham},
  booktitle={2015 IEEE 31St international conference on data engineering},
  pages={1023--1034},
  year={2015},
  organization={IEEE}
}

@article{alaggan2016heterogeneous,
  title={Heterogeneous Differential Privacy},
  author={Alaggan, Mohammad and Gambs, S{\'e}bastien and Kermarrec, Anne-Marie},
  journal={Journal of Privacy and Confidentiality},
  volume={7},
  number={2},
  year={2016}
}

@inproceedings{allen2019convergence,
  title={A convergence theory for deep learning via over-parameterization},
  author={Allen-Zhu, Zeyuan and Li, Yuanzhi and Song, Zhao},
  booktitle={International Conference on Machine Learning},
  pages={242--252},
  year={2019},
  organization={PMLR}
}

@inproceedings{arora2018optimization,
  title={On the optimization of deep networks: Implicit acceleration by overparameterization},
  author={Arora, Sanjeev and Cohen, Nadav and Hazan, Elad},
  booktitle={International Conference on Machine Learning},
  pages={244--253},
  year={2018},
  organization={PMLR}
}

@article{jacot2018neural,
  title={Neural tangent kernel: Convergence and generalization in neural networks},
  author={Jacot, Arthur and Gabriel, Franck and Hongler, Cl{\'e}ment},
  journal={arXiv preprint arXiv:1806.07572},
  year={2018}
}

@inproceedings{brown2021memorization,
  title={When is memorization of irrelevant training data necessary for high-accuracy learning?},
  author={Brown, Gavin and Bun, Mark and Feldman, Vitaly and Smith, Adam and Talwar, Kunal},
  booktitle={Proceedings of the 53rd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={123--132},
  year={2021}
}

@inproceedings{feldman2020does,
  title={Does learning require memorization? a short tale about a long tail},
  author={Feldman, Vitaly},
  booktitle={Proceedings of the 52nd Annual ACM SIGACT Symposium on Theory of Computing},
  pages={954--959},
  year={2020}
}

@article{song2020evading,
  title={Evading Curse of Dimensionality in Unconstrained Private GLMs via Private Gradient Descent},
  author={Song, Shuang and Steinke, Thomas and Thakkar, Om and Thakurta, Abhradeep},
  journal={arXiv preprint arXiv:2006.06783},
  year={2020}
}

@inproceedings{zhou2020bypassing,
  title={Bypassing the Ambient Dimension: Private SGD with Gradient Subspace Identification},
  author={Zhou, Yingxue and Wu, Steven and Banerjee, Arindam},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{kairouz2021nearly,
  title={(Nearly) Dimension Independent Private ERM with AdaGrad Rates via Publicly Estimated Subspaces},
  author={Kairouz, Peter and Diaz, Monica Ribero and Rush, Keith and Thakurta, Abhradeep},
  booktitle={Conference on Learning Theory},
  pages={2717--2746},
  year={2021},
  organization={PMLR}
}

@article{zhang2020adaptive,
  title={Why are Adaptive Methods Good for Attention Models?},
  author={Zhang, Jingzhao and Karimireddy, Sai Praneeth and Veit, Andreas and Kim, Seungyeon and Reddi, Sashank and Kumar, Sanjiv and Sra, Suvrit},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  year={2020}
}

@inproceedings{simsekli2019tail,
  title={A tail-index analysis of stochastic gradient noise in deep neural networks},
  author={Simsekli, Umut and Sagun, Levent and Gurbuzbalaban, Mert},
  booktitle={International Conference on Machine Learning},
  pages={5827--5837},
  year={2019},
  organization={PMLR}
}

@article{bubeck2013bandits,
  title={Bandits with heavy tail},
  author={Bubeck, S{\'e}bastien and Cesa-Bianchi, Nicolo and Lugosi, G{\'a}bor},
  journal={IEEE Transactions on Information Theory},
  volume={59},
  number={11},
  pages={7711--7717},
  year={2013},
  publisher={IEEE}
}


@article{lugosi2019mean,
  title={Mean estimation and regression under heavy-tailed distributions: A survey},
  author={Lugosi, G{\'a}bor and Mendelson, Shahar},
  journal={Foundations of Computational Mathematics},
  volume={19},
  number={5},
  pages={1145--1190},
  year={2019},
  publisher={Springer}
}

@article{catoni2017dimension,
  title={Dimension-free PAC-Bayesian bounds for matrices, vectors, and linear least squares regression},
  author={Catoni, Olivier and Giulini, Ilaria},
  journal={arXiv preprint arXiv:1712.02747},
  year={2017}
}

@inproceedings{kamath2020private,
  title={Private mean estimation of heavy-tailed distributions},
  author={Kamath, Gautam and Singhal, Vikrant and Ullman, Jonathan},
  booktitle={Conference on Learning Theory},
  pages={2204--2235},
  year={2020},
  organization={PMLR}
}

@inproceedings{loshchilov2018decoupled,
  title={Decoupled Weight Decay Regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{li2019convergence,
  title={On the convergence of stochastic gradient descent with adaptive stepsizes},
  author={Li, Xiaoyu and Orabona, Francesco},
  booktitle={The 22nd International Conference on Artificial Intelligence and Statistics},
  pages={983--992},
  year={2019},
  organization={PMLR}
}

@inproceedings{ward2019adagrad,
  title={AdaGrad stepsizes: Sharp convergence over nonconvex landscapes},
  author={Ward, Rachel and Wu, Xiaoxia and Bottou, Leon},
  booktitle={International Conference on Machine Learning},
  pages={6677--6686},
  year={2019},
  organization={PMLR}
}

@article{brown2021covariance,
  title={Covariance-Aware Private Mean Estimation Without Private Covariance Estimation},
  author={Brown, Gavin and Gaboardi, Marco and Smith, Adam and Ullman, Jonathan and Zakynthinou, Lydia},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}



@article{tang2017privacy,
  title={Privacy loss in apple's implementation of differential privacy on macos 10.12},
  author={Tang, Jun and Korolova, Aleksandra and Bai, Xiaolong and Wang, Xueqiang and Wang, Xiaofeng},
  journal={arXiv preprint arXiv:1709.02753},
  year={2017}
}


@software{jax2018github,
  author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and George Necula and Adam Paszke and Jake Vander{P}las and Skye Wanderman-{M}ilne and Qiao Zhang},
  title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs},
  url = {http://github.com/google/jax},
  version = {0.2.5},
  year = {2018},
}

@misc{tensorflowprivacy2021github,
  author = {Google L.L.C.},
  title = {Tensorflow Privacy},
  year = {2021},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/tensorflow/privacy}}
}

@inproceedings{mehta2020extreme,
  title={Extreme Memorization via Scale of Initialization},
  author={Mehta, Harsh and Cutkosky, Ashok and Neyshabur, Behnam},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{bhaskara2020online,
  title={Online learning with imperfect hints},
  author={Bhaskara, Aditya and Cutkosky, Ashok and Kumar, Ravi and Purohit, Manish},
  booktitle={International Conference on Machine Learning},
  pages={822--831},
  year={2020},
  organization={PMLR}
}

@article{bhaskara2021logarithmic,
  title={Logarithmic Regret from Sublinear Hints},
  author={Bhaskara, Aditya and Cutkosky, Ashok and Kumar, Ravi and Purohit, Manish},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021},
  pages={28222--28232}
}


@inproceedings{alieva2021robust,
  title={Robust pure exploration in linear bandits with limited budget},
  author={Alieva, Ayya and Cutkosky, Ashok and Das, Abhimanyu},
  booktitle={International Conference on Machine Learning},
  pages={187--195},
  year={2021},
  organization={PMLR}
}


@inproceedings{cutkosky2021dynamic,
  title={Dynamic balancing for model selection in bandits and rl},
  author={Cutkosky, Ashok and Dann, Christoph and Das, Abhimanyu and Gentile, Claudio and Pacchiano, Aldo and Purohit, Manish},
  booktitle={International Conference on Machine Learning},
  pages={2276--2285},
  year={2021},
  organization={PMLR}
}


@inproceedings{rakhlin2017equivalence,
  title={On equivalence of martingale tail bounds and deterministic regret inequalities},
  author={Rakhlin, Alexander and Sridharan, Karthik},
  booktitle={Conference on Learning Theory},
  pages={1704--1722},
  year={2017},
  organization={PMLR}
}

@article{subramani2021enabling,
  title={Enabling fast differentially private sgd via just-in-time compilation and vectorization},
  author={Subramani, Pranav and Vadivelu, Nicholas and Kamath, Gautam},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@inproceedings{andrew2021differentially,
  title={Differentially private learning with adaptive clipping},
  author={Andrew, Galen and Thakkar, Om and Ramaswamy, Swaroop and McMahan, Hugh Brendan},
  booktitle={Thirty-Fifth Conference on Neural Information Processing Systems},
  year={2021}
}

@inproceedings{amin2019bounding,
  title={Bounding user contributions: A bias-variance trade-off in differential privacy},
  author={Amin, Kareem and Kulesza, Alex and Munoz, Andres and Vassilvtiskii, Sergei},
  booktitle={International Conference on Machine Learning},
  pages={263--271},
  year={2019},
  organization={PMLR}
}


@inproceedings{bernstein2018signsgd,
  title={signSGD: Compressed optimisation for non-convex problems},
  author={Bernstein, Jeremy and Wang, Yu-Xiang and Azizzadenesheli, Kamyar and Anandkumar, Animashree},
  booktitle={International Conference on Machine Learning},
  pages={560--569},
  year={2018},
  organization={PMLR}
}
@inproceedings{karimireddy2019error,
  title={Error feedback fixes signsgd and other gradient compression schemes},
  author={Karimireddy, Sai Praneeth and Rebjock, Quentin and Stich, Sebastian and Jaggi, Martin},
  booktitle={International Conference on Machine Learning},
  pages={3252--3261},
  year={2019},
  organization={PMLR}
}


@article{you2019large,
  title={Large batch optimization for deep learning: Training bert in 76 minutes},
  author={You, Yang and Li, Jing and Reddi, Sashank and Hseu, Jonathan and Kumar, Sanjiv and Bhojanapalli, Srinadh and Song, Xiaodan and Demmel, James and Keutzer, Kurt and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:1904.00962},
  year={2019}
}


@article{you2017scaling,
  title={Scaling sgd batch size to 32k for imagenet training},
  author={You, Yang and Gitman, Igor and Ginsburg, Boris},
  journal={arXiv preprint arXiv:1708.03888},
  volume={6},
  pages={12},
  year={2017}
}


@article{nado2021large,
  title={A large batch optimizer reality check: Traditional, generic optimizers suffice across batch sizes},
  author={Nado, Zachary and Gilmer, Justin M and Shallue, Christopher J and Anil, Rohan and Dahl, George E},
  journal={arXiv preprint arXiv:2102.06356},
  year={2021}
}

@article{bubeck2015convex,
  title={Convex Optimization: Algorithms and Complexity},
  author={Bubeck, S{\'e}bastien},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={8},
  number={3-4},
  pages={231--357},
  year={2015},
  publisher={Now Publishers, Inc.}
}


@article{nesterovintroductory,
  title={Introductory Lectures on Convex Optimization A Basic Course},
  author={Nesterov, Yurii},
  publisher={Springer}
}


@article{nesterov2006cubic,
  title={Cubic regularization of Newton method and its global performance},
  author={Nesterov, Yurii and Polyak, Boris T},
  journal={Mathematical Programming},
  volume={108},
  number={1},
  pages={177--205},
  year={2006},
  publisher={Springer}
}

@inproceedings{carmon2017convex,
  title={“Convex Until Proven Guilty”: Dimension-Free Acceleration of Gradient Descent on Non-Convex Functions},
  author={Carmon, Yair and Duchi, John C and Hinder, Oliver and Sidford, Aaron},
  booktitle={International Conference on Machine Learning},
  pages={654--663},
  year={2017},
  organization={PMLR}
}


@article{carmon2021lower,
  title={Lower bounds for finding stationary points II: first-order methods},
  author={Carmon, Yair and Duchi, John C and Hinder, Oliver and Sidford, Aaron},
  journal={Mathematical Programming},
  volume={185},
  number={1-2},
  year={2021}
}


@article{goyal2017accurate,
  title={Accurate, large minibatch sgd: Training imagenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={arXiv preprint arXiv:1706.02677},
  year={2017}
}


@article{loshchilov2016sgdr,
  title={Sgdr: Stochastic gradient descent with warm restarts},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1608.03983},
  year={2016}
}


@article{nguyen2020unified,
  title={A Unified Convergence Analysis for Shuffling-Type Gradient Methods},
  author={Nguyen, Lam M and Tran-Dinh, Quoc and Phan, Dzung T and Nguyen, Phuong Ha and van Dijk, Marten},
  journal={arXiv preprint arXiv:2002.08246},
  year={2020}
}


@article{mishchenko2020random,
  title={Random Reshuffling: Simple Analysis with Vast Improvements},
  author={Mishchenko, Konstantin and Khaled, Ahmed and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2006.05988},
  year={2020}
}


@inproceedings{niu2011hogwild,
  title={HOGWILD! a lock-free approach to parallelizing stochastic gradient descent},
  author={Niu, Feng and Recht, Benjamin and Re, Christopher and Wright, Stephen J},
  booktitle={Proceedings of the 24th International Conference on Neural Information Processing Systems},
  pages={693--701},
  year={2011}
}

@inproceedings{reddi2018convergence,
  author    = {Sashank J. Reddi and
               Satyen Kale and
               Sanjiv Kumar},
  title     = {On the Convergence of Adam and Beyond},
  booktitle = {6th International Conference on Learning Representations, {ICLR} 2018},
  year      = {2018},
}


@article{shalev2013stochastic,
  title={Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.},
  author={Shalev-Shwartz, Shai and Zhang, Tong},
  journal={Journal of Machine Learning Research},
  volume={14},
  number={2},
  year={2013}
}


@inproceedings{roux2012stochastic,
  title={A stochastic gradient method with an exponential convergence rate for finite training sets},
  author={Roux, Nicolas Le and Schmidt, Mark and Bach, Francis},
  booktitle={Proceedings of the 25th International Conference on Neural Information Processing Systems-Volume 2},
  pages={2663--2671},
  year={2012}
}


@article{carmon2019lower,
  title={Lower bounds for finding stationary points i},
  author={Carmon, Yair and Duchi, John C and Hinder, Oliver and Sidford, Aaron},
  journal={Mathematical Programming},
  pages={1--50},
  year={2019},
  publisher={Springer}
}


@article{baydin2018automatic,
  title={Automatic differentiation in machine learning: a survey},
  author={Baydin, Atilim Gunes and Pearlmutter, Barak A and Radul, Alexey Andreyevich and Siskind, Jeffrey Mark},
  journal={Journal of machine learning research},
  volume={18},
  year={2018},
  publisher={Journal of Machine Learning Research}
}

@article{rumelhart1986learning,
  title={Learning representations by back-propagating errors},
  author={Rumelhart, David E and Hinton, Geoffrey E and Williams, Ronald J},
  journal={nature},
  volume={323},
  number={6088},
  pages={533--536},
  year={1986},
  publisher={Nature Publishing Group}
}

@inproceedings{mironov2017renyi,
  title={R{\'e}nyi differential privacy},
  author={Mironov, Ilya},
  booktitle={2017 IEEE 30th Computer Security Foundations Symposium (CSF)},
  pages={263--275},
  year={2017},
  organization={IEEE}
}

@book{tu2017differential,
  title={Differential geometry: connections, curvature, and characteristic classes},
  author={Tu, Loring W},
  volume={275},
  year={2017},
  publisher={Springer}
}

@book{tu2011introduction,
  title={An introduction to manifolds},
  author={Loring W.. Tu},
  year={2011},
  publisher={Springer.}
}

@book{bott1982differential,
  title={Differential forms in algebraic topology},
  author={Bott, Raoul and Tu, Loring W and others},
  volume={82},
  year={1982},
  publisher={Springer}
}

@inproceedings{daniely2015strongly,
  title={Strongly adaptive online learning},
  author={Daniely, Amit and Gonen, Alon and Shalev-Shwartz, Shai},
  booktitle={International Conference on Machine Learning},
  pages={1405--1411},
  year={2015},
  organization={PMLR}
}
@article{chaudhuri2011differentially,
  title={Differentially private empirical risk minimization.},
  author={Chaudhuri, Kamalika and Monteleoni, Claire and Sarwate, Anand D},
  journal={Journal of Machine Learning Research},
  volume={12},
  number={3},
  year={2011}
}
@inproceedings{bassily2014private,
  title={Private empirical risk minimization: Efficient algorithms and tight error bounds},
  author={Bassily, Raef and Smith, Adam and Thakurta, Abhradeep},
  booktitle={2014 IEEE 55th Annual Symposium on Foundations of Computer Science},
  pages={464--473},
  year={2014},
  organization={IEEE}
}

@inproceedings{kifer2012private,
  title={Private convex empirical risk minimization and high-dimensional regression},
  author={Kifer, Daniel and Smith, Adam and Thakurta, Abhradeep},
  booktitle={Conference on Learning Theory},
  pages={25--1},
  year={2012},
  organization={JMLR Workshop and Conference Proceedings}
}

@InProceedings{pmlr-v23-kifer12,
  title = 	 {Private Convex Empirical Risk Minimization and High-dimensional Regression},
  author = 	 {Kifer, Daniel and Smith, Adam and Thakurta, Abhradeep},
  booktitle = 	 {Proceedings of the 25th Annual Conference on Learning Theory},
  pages = 	 {25.1--25.40},
  year = 	 {2012},
  editor = 	 {Mannor, Shie and Srebro, Nathan and Williamson, Robert C.},
  volume = 	 {23},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Edinburgh, Scotland},
  month = 	 {25--27 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v23/kifer12/kifer12.pdf},
  url = 	 {https://proceedings.mlr.press/v23/kifer12.html},
  abstract = 	 {We consider \emphdifferentially private algorithms for convex empirical risk minimization (ERM). Differential privacy (Dwork et al., 2006b) is a recently introduced notion of privacy which guarantees that an algorithm’s output does not depend on the data of any individual in the dataset. This is crucial in fields that handle sensitive data, such as genomics, collaborative filtering, and economics. Our motivation is the design of private algorithms for sparse learning problems, in which one aims to find solutions (e.g., regression parameters) with few non-zero coefficients. To this end: (a) We significantly extend the analysis of the “objective perturbation” algorithm of Chaudhuri et al. (2011) for convex ERM problems. We show that their method can be modified to use less noise (be more accurate), and to apply to problems with hard constraints and non-differentiable regularizers. We also give a tighter, data-dependent analysis of the additional error introduced by their method. A key tool in our analysis is a new nontrivial limit theorem for differential privacy which is of independent interest: if a sequence of differentially private algorithms converges, in a \emphweak sense, then the limit algorithm is also differentially private. In particular, our methods give the best known algorithms for differentially private linear regression. These methods work in settings where the number of parameters p is less than the number of samples n. (b) We give the first two private algorithms for \emphsparse regression problems in high-dimensional settings, where p is much larger than n. We analyze their performance for linear regression: under standard assumptions on the data, our algorithms have vanishing empirical risk for n = poly(s, \log p) when there exists a good regression vector with s nonzero coefficients. Our algorithms demonstrate that randomized algorithms for sparse regression problems can be both stable and accurate - a combination which is impossible for deterministic algorithms.}
}

@INPROCEEDINGS{6736861,
  author={Song, Shuang and Chaudhuri, Kamalika and Sarwate, Anand D.},
  booktitle={2013 IEEE Global Conference on Signal and Information Processing}, 
  title={Stochastic gradient descent with differentially private updates}, 
  year={2013},
  volume={},
  number={},
  pages={245-248},
  doi={10.1109/GlobalSIP.2013.6736861}}
 
 
@InProceedings{pmlr-v32-jain14,
  title = 	 {(Near) Dimension Independent Risk Bounds for Differentially Private Learning},
  author = 	 {Jain, Prateek and Thakurta, Abhradeep Guha},
  booktitle = 	 {Proceedings of the 31st International Conference on Machine Learning},
  pages = 	 {476--484},
  year = 	 {2014},
  editor = 	 {Xing, Eric P. and Jebara, Tony},
  volume = 	 {32},
  number =       {1},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Bejing, China},
  month = 	 {22--24 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v32/jain14.pdf},
  url = 	 {https://proceedings.mlr.press/v32/jain14.html},
  abstract = 	 {In this paper, we study the problem of differentially private risk minimization where the goal is to provide differentially private algorithms that have small excess risk. In particular we address the following open problem: \emphIs it possible to design computationally efficient differentially private risk minimizers with  excess risk bounds that do not explicitly depend on dimensionality (p) and do not require  structural assumptions like restricted strong convexity?  In this paper, we answer the question in the affirmative for a variant of the well-known  \emphoutput and \emphobjective perturbation algorithms [Chaudhuri et al., 2011]. In particular, we show that  in generalized linear model, variants of both output and objective perturbation algorithms have no \em explicit dependence on p. Our results assume that the underlying loss function is a 1-Lipschitz convex function and we show that the excess risk depends only on  L_2 norm of the true risk minimizer and that of training points.  Next, we present a novel privacy preserving algorithm for risk minimization over simplex in the generalized linear model, where the loss function is  a doubly differentiable convex function. Assuming that the training points have bounded L_∞-norm, our algorithm provides risk bound that has only \em logarithmic dependence on p. We also apply our technique to the online learning setting and obtain a regret bound with similar logarithmic dependence on p. In contrast, the existing differentially private online learning methods incur O(\sqrtp)  dependence.}
}
@inproceedings{NIPS2015_52d080a3,
 author = {Talwar, Kunal and Guha Thakurta, Abhradeep and Zhang, Li},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {C. Cortes and N. Lawrence and D. Lee and M. Sugiyama and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Nearly Optimal Private LASSO},
 url = {https://proceedings.neurips.cc/paper/2015/file/52d080a3e172c33fd6886a37e7288491-Paper.pdf},
 volume = {28},
 year = {2015}
}

@inproceedings{wu2017bolt,
  title={Bolt-on differential privacy for scalable stochastic gradient descent-based analytics},
  author={Wu, Xi and Li, Fengan and Kumar, Arun and Chaudhuri, Kamalika and Jha, Somesh and Naughton, Jeffrey},
  booktitle={Proceedings of the 2017 ACM International Conference on Management of Data},
  pages={1307--1322},
  year={2017}
}

@InProceedings{pmlr-v89-chen19e,
  title = 	 {Renyi Differentially Private ERM for Smooth Objectives},
  author =       {Chen, Chen and Lee, Jaewoo and Kifer, Dan},
  booktitle = 	 {Proceedings of the Twenty-Second International Conference on Artificial Intelligence and Statistics},
  pages = 	 {2037--2046},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Sugiyama, Masashi},
  volume = 	 {89},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {16--18 Apr},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v89/chen19e/chen19e.pdf},
  url = 	 {https://proceedings.mlr.press/v89/chen19e.html},
  abstract = 	 {In this paper, we present a Renyi Differentially Private stochastic gradient descent (SGD) algorithm for convex empirical risk minimization. The algorithm uses output perturbation and leverages randomness inside SGD, which creates a "randomized sensitivity", in order to reduce the amount of noise that is added. One of the benefits of output perturbation is that we can incorporate a periodic averaging step that  serves to further reduce sensitivity while improving accuracy (reducing the well-known oscillating behavior of SGD near the optimum).  Renyi Differential Privacy can be used to provide (epsilon, delta)-differential privacy guarantees and hence provide a comparison with prior work. An empirical evaluation demonstrates that the proposed method outperforms prior methods on differentially private ERM.}
}
@article{wang2018differentially,
  title={Differentially private empirical risk minimization revisited: Faster and more general},
  author={Wang, Di and Ye, Minwei and Xu, Jinhui},
  journal={arXiv preprint arXiv:1802.05251},
  year={2018}
}

@inproceedings{song2021evading,
  title={Evading the curse of dimensionality in unconstrained private glms},
  author={Song, Shuang and Steinke, Thomas and Thakkar, Om and Thakurta, Abhradeep},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={2638--2646},
  year={2021},
  organization={PMLR}
}
@article{bassily2021differentially,
  title={Differentially private stochastic optimization: New results in convex and non-convex settings},
  author={Bassily, Raef and Guzm{\'a}n, Crist{\'o}bal and Menart, Michael},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{dwork2015generalization,
  title={Generalization in adaptive data analysis and holdout reuse},
  author={Dwork, Cynthia and Feldman, Vitaly and Hardt, Moritz and Pitassi, Toniann and Reingold, Omer and Roth, Aaron},
  journal={arXiv preprint arXiv:1506.02629},
  year={2015}
}
@inproceedings{dwork2015preserving,
  title={Preserving statistical validity in adaptive data analysis},
  author={Dwork, Cynthia and Feldman, Vitaly and Hardt, Moritz and Pitassi, Toniann and Reingold, Omer and Roth, Aaron Leon},
  booktitle={Proceedings of the forty-seventh annual ACM symposium on Theory of computing},
  pages={117--126},
  year={2015}
}

@inproceedings{jun2019parameter,
  title={Parameter-free online convex optimization with sub-exponential noise},
  author={Jun, Kwang-Sung and Orabona, Francesco},
  booktitle={Conference on Learning Theory},
  pages={1802--1823},
  year={2019},
  organization={PMLR}
}

@article{orabona2021tight,
  title={Tight Concentrations and Confidence Sequences from the Regret of Universal Portfolio},
  author={Orabona, Francesco and Jun, Kwang-Sung},
  journal={arXiv preprint arXiv:2110.14099},
  year={2021}
}

@inproceedings{10.1145/1242572.1242598,
author = {Backstrom, Lars and Dwork, Cynthia and Kleinberg, Jon},
title = {Wherefore Art Thou R3579x? Anonymized Social Networks, Hidden Patterns, and Structural Steganography},
year = {2007},
isbn = {9781595936547},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/1242572.1242598},
doi = {10.1145/1242572.1242598},
abstract = {In a social network, nodes correspond topeople or other social entities, and edges correspond to social links between them. In an effort to preserve privacy, the practice of anonymization replaces names with meaningless unique identifiers. We describe a family of attacks such that even from a single anonymized copy of a social network, it is possible for an adversary to learn whether edges exist or not between specific targeted pairs of nodes.},
booktitle = {Proceedings of the 16th International Conference on World Wide Web},
pages = {181–190},
numpages = {10},
keywords = {social networks, privacy in data mining, anonymization},
location = {Banff, Alberta, Canada},
series = {WWW '07}
}

@inproceedings{10.1145/773153.773173,
author = {Dinur, Irit and Nissim, Kobbi},
title = {Revealing Information While Preserving Privacy},
year = {2003},
isbn = {1581136706},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/773153.773173},
doi = {10.1145/773153.773173},
abstract = {We examine the tradeoff between privacy and usability of statistical databases. We model a statistical database by an n-bit string d1,..,dn, with a query being a subset q ⊆ [n] to be answered by Σiεq di. Our main result is a polynomial reconstruction algorithm of data from noisy (perturbed) subset sums. Applying this reconstruction algorithm to statistical databases we show that in order to achieve privacy one has to add perturbation of magnitude (Ω√n). That is, smaller perturbation always results in a strong violation of privacy. We show that this result is tight by exemplifying access algorithms for statistical databases that preserve privacy while adding perturbation of magnitude \~{O}(√n).For time-T bounded adversaries we demonstrate a privacypreserving access algorithm whose perturbation magnitude is ≈ √T.},
booktitle = {Proceedings of the Twenty-Second ACM SIGMOD-SIGACT-SIGART Symposium on Principles of Database Systems},
pages = {202–210},
numpages = {9},
keywords = {subset-sums with noise, integrity and security, data reconstruction},
location = {San Diego, California},
series = {PODS '03}
}

@inproceedings{10.1007/11681878_14,
author = {Dwork, Cynthia and McSherry, Frank and Nissim, Kobbi and Smith, Adam},
title = {Calibrating Noise to Sensitivity in Private Data Analysis},
year = {2006},
isbn = {3540327312},
publisher = {Springer-Verlag},
address = {Berlin, Heidelberg},
url = {https://doi.org/10.1007/11681878_14},
doi = {10.1007/11681878_14},
abstract = {We continue a line of research initiated in [10,11]on privacy-preserving statistical databases. Consider a trusted server that holds a database of sensitive information. Given a query function f mapping databases to reals, the so-called true answer is the result of applying f to the database. To protect privacy, the true answer is perturbed by the addition of random noise generated according to a carefully chosen distribution, and this response, the true answer plus noise, is returned to the user.Previous work focused on the case of noisy sums, in which f = ∑ig(xi), where xi denotes the ith row of the database and g maps database rows to [0,1]. We extend the study to general functions f, proving that privacy can be preserved by calibrating the standard deviation of the noise according to the sensitivity of the function f. Roughly speaking, this is the amount that any single argument to f can change its output. The new analysis shows that for several particular applications substantially less noise is needed than was previously understood to be the case.The first step is a very clean characterization of privacy in terms of indistinguishability of transcripts. Additionally, we obtain separation results showing the increased value of interactive sanitization mechanisms over non-interactive.},
booktitle = {Proceedings of the Third Conference on Theory of Cryptography},
pages = {265–284},
numpages = {20},
location = {New York, NY},
series = {TCC'06}
}

@INPROCEEDINGS{8835258,
  author={Iyengar, Roger and Near, Joseph P. and Song, Dawn and Thakkar, Om and Thakurta, Abhradeep and Wang, Lun},
  booktitle={2019 IEEE Symposium on Security and Privacy (SP)}, 
  title={Towards Practical Differentially Private Convex Optimization}, 
  year={2019},
  volume={},
  number={},
  pages={299-316},
  doi={10.1109/SP.2019.00001}}

@article{talwar2014private,
  title={Private empirical risk minimization beyond the worst case: The effect of the constraint set geometry},
  author={Talwar, Kunal and Thakurta, Abhradeep and Zhang, Li},
  journal={arXiv preprint arXiv:1411.5417},
  year={2014}
}

@inproceedings{NEURIPS2018_7221e5c8,
 author = {Jayaraman, Bargav and Wang, Lingxiao and Evans, David and Gu, Quanquan},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {S. Bengio and H. Wallach and H. Larochelle and K. Grauman and N. Cesa-Bianchi and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Distributed Learning without Distress: Privacy-Preserving Empirical Risk Minimization},
 url = {https://proceedings.neurips.cc/paper/2018/file/7221e5c8ec6b08ef6d3f9ff3ce6eb1d1-Paper.pdf},
 volume = {31},
 year = {2018}
}

@article{balle2018privacy,
  title={Privacy amplification by subsampling: Tight analyses via couplings and divergences},
  author={Balle, Borja and Barthe, Gilles and Gaboardi, Marco},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}
@article{kulkarni2021private,
  title={Private non-smooth empirical risk minimization and stochastic convex optimization in subquadratic steps},
  author={Kulkarni, Janardhan and Lee, Yin Tat and Liu, Daogao},
  journal={arXiv preprint arXiv:2103.15352},
  year={2021}
}

@article{defazio2019ineffectiveness,
  title={On the ineffectiveness of variance reduced optimization for deep learning},
  author={Defazio, Aaron and Bottou, L{\'e}on},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}

@inproceedings{srinivas2010gaussian,
  title={Gaussian process optimization in the bandit setting: no regret and experimental design},
  author={Srinivas, Niranjan and Krause, Andreas and Kakade, Sham and Seeger, Matthias},
  booktitle={Proceedings of the 27th International Conference on International Conference on Machine Learning},
  pages={1015--1022},
  year={2010}
}

@article{you2019large,
  title={Large batch optimization for deep learning: Training bert in 76 minutes},
  author={You, Yang and Li, Jing and Reddi, Sashank and Hseu, Jonathan and Kumar, Sanjiv and Bhojanapalli, Srinadh and Song, Xiaodan and Demmel, James and Keutzer, Kurt and Hsieh, Cho-Jui},
  journal={arXiv preprint arXiv:1904.00962},
  year={2019}
}

@article{you2017large,
  title={Large batch training of convolutional networks},
  author={You, Yang and Gitman, Igor and Ginsburg, Boris},
  journal={arXiv preprint arXiv:1708.03888},
  year={2017}
}


@inproceedings{asi2021private,
  title={Private adaptive gradient methods for convex optimization},
  author={Asi, Hilal and Duchi, John and Fallah, Alireza and Javidbakht, Omid and Talwar, Kunal},
  booktitle={International Conference on Machine Learning},
  pages={383--392},
  year={2021},
  organization={PMLR}
}

@article{asi2020near,
  title={Near instance-optimality in differential privacy},
  author={Asi, Hilal and Duchi, John C},
  journal={arXiv preprint arXiv:2005.10630},
  year={2020}
}

@article{asi2021adapting,
  title={Adapting to function difficulty and growth conditions in private optimization},
  author={Asi, Hilal and Levy, Daniel and Duchi, John C},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{li2022private,
  title={Private Adaptive Optimization with Side Information},
  author={Li, Tian and Zaheer, Manzil and Reddi, Sashank J and Smith, Virginia},
  journal={arXiv preprint arXiv:2202.05963},
  year={2022}
}

@inproceedings{orabona2020tutorial,
  author = {Orabona, Francesco and Cutkosky, Ashok},
  booktitle = {International Conference on Machine Learning (ICML)},
  title = {Tutorial on Parameter-free Online Optimization},
  url = {https://parameterfree.com/icml-tutorial/},
  year = {2020},
}
@article{nguyen2019tight,
  title={Tight dimension independent lower bound on the expected convergence rate for diminishing step sizes in SGD},
  author={Nguyen, Phuong Ha and Nguyen, Lam M and van Dijk, Marten},
  journal={Advances in Neural Information Processing Systems},
  volume={32},
  year={2019}
}
@article{tran2021better,
  title={Better SGD using Second-order Momentum},
  author={Tran, Hoang and Cutkosky, Ashok},
  journal={arXiv preprint arXiv:2103.03265},
  year={2021}
}

@article {Jiang230,
	author = {Jiang, Fei and Jiang, Yong and Zhi, Hui and Dong, Yi and Li, Hao and Ma, Sufeng and Wang, Yilong and Dong, Qiang and Shen, Haipeng and Wang, Yongjun},
	title = {Artificial intelligence in healthcare: past, present and future},
	volume = {2},
	number = {4},
	pages = {230--243},
	year = {2017},
	doi = {10.1136/svn-2017-000101},
	publisher = {BMJ Specialist Journals},
	issn = {2059-8688},
	URL = {https://svn.bmj.com/content/2/4/230},
	eprint = {https://svn.bmj.com/content/2/4/230.full.pdf},
	journal = {Stroke and Vascular Neurology}
}

@article{article,
author = {He, Jianxing and Baxter, Sally and Xu, Jie and Xu, Jiming and Zhou, Xingtao and Zhang, Kang},
year = {2019},
month = {01},
pages = {},
title = {The practical implementation of artificial intelligence technologies in medicine},
volume = {25},
journal = {Nature Medicine},
doi = {10.1038/s41591-018-0307-0}
}
@article{chen2021applications,
  title={Applications of artificial intelligence in drug development using real-world data},
  author={Chen, Zhaoyi and Liu, Xiong and Hogan, William and Shenkman, Elizabeth and Bian, Jiang},
  journal={Drug discovery today},
  volume={26},
  number={5},
  pages={1256--1264},
  year={2021},
  publisher={Elsevier}
}

@article{howard2021time,
  title={Time-uniform, nonparametric, nonasymptotic confidence sequences},
  author={Howard, Steven R and Ramdas, Aaditya and McAuliffe, Jon and Sekhon, Jasjeet},
  journal={The Annals of Statistics},
  volume={49},
  number={2},
  pages={1055--1080},
  year={2021},
  publisher={Institute of Mathematical Statistics}
}

@article{balsubramani2014sharp,
  title={Sharp finite-time iterated-logarithm martingale concentration},
  author={Balsubramani, Akshay},
  journal={arXiv preprint arXiv:1405.2639},
  year={2014}
}

@phdthesis{cutkosky2018algorithms,
  title={Algorithms and Lower Bounds for Parameter-free Online Learning},
  author={Cutkosky, Ashok},
  year={2018},
  school={Stanford University}
}

@inproceedings{zhuang2021no,
  title={No-Regret Reinforcement Learning with Heavy-Tailed Rewards},
  author={Zhuang, Vincent and Sui, Yanan},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={3385--3393},
  year={2021},
  organization={PMLR}
}

@misc{koolen,
title={A Quick and Dirty Finite Time Law of the Iterated Logarithm Result}, url={http://blog.wouterkoolen.info/QnD_LIL/post.html}, 
author={Koolen, Wouter  M}} 


@book{folland1999real,
  title={Real Analysis: Modern Techniques and Their Applications},
  author={Folland, Gerald B.},
  isbn={9780471317166},
  lccn={98037260},
  series={A Wiley-Interscience publication},
  url={https://books.google.com/books?id=uPkYAQAAIAAJ},
  year={1999},
  publisher={Wiley}
}

@book{bauschke2011convex,
  title={Convex analysis and monotone operator theory in Hilbert spaces},
  author={Bauschke, Heinz H and Combettes, Patrick L and others},
  volume={408},
  year={2011},
  publisher={Springer}
}

@article{jin2019short,
  title={A short note on concentration inequalities for random vectors with subgaussian norm},
  author={Jin, Chi and Netrapalli, Praneeth and Ge, Rong and Kakade, Sham M and Jordan, Michael I},
  journal={arXiv preprint arXiv:1902.03736},
  year={2019}
}

@inproceedings{agarwal2019online,
  title={Online control with adversarial disturbances},
  author={Agarwal, Naman and Bullins, Brian and Hazan, Elad and Kakade, Sham and Singh, Karan},
  booktitle={International Conference on Machine Learning},
  pages={111--119},
  year={2019},
  organization={PMLR}
}


@inproceedings{foster2020logarithmic,
  title={Logarithmic regret for adversarial online control},
  author={Foster, Dylan and Simchowitz, Max},
  booktitle={International Conference on Machine Learning},
  pages={3211--3221},
  year={2020},
  organization={PMLR}
}


@inproceedings{zhang2022adversarial,
  title={Adversarial tracking control via strongly adaptive online learning with memory},
  author={Zhang, Zhiyu and Cutkosky, Ashok and Paschalidis, Ioannis},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={8458--8492},
  year={2022},
  organization={PMLR}
}

@article{vovk2007hoeffding,
  title={Hoeffding's inequality in game-theoretic probability},
  author={Vovk, Vladimir},
  journal={arXiv preprint arXiv:0708.2502},
  year={2007}
}

@article{waudby2020estimating,
  title={Estimating means of bounded random variables by betting},
  author={Waudby-Smith, Ian and Ramdas, Aaditya},
  journal={arXiv preprint arXiv:2010.09686},
  year={2020}
}

@inproceedings{chen2021impossible,
  title={Impossible tuning made possible: A new expert algorithm and its applications},
  author={Chen, Liyu and Luo, Haipeng and Wei, Chen-Yu},
  booktitle={Conference on Learning Theory},
  pages={1216--1259},
  year={2021},
  organization={PMLR}
}

@inproceedings{dwork2006calibrating,
  title={Calibrating noise to sensitivity in private data analysis},
  author={Dwork, Cynthia and McSherry, Frank and Nissim, Kobbi and Smith, Adam},
  booktitle={Theory of cryptography conference},
  pages={265--284},
  year={2006},
  organization={Springer}
}

@article{kurakin2022toward,
  title={Toward training at imagenet scale with differential privacy},
  author={Kurakin, Alexey and Chien, Steve and Song, Shuang and Geambasu, Roxana and Terzis, Andreas and Thakurta, Abhradeep},
  journal={arXiv preprint arXiv:2201.12328},
  year={2022}
}

@inproceedings{hoory2021learning,
  title={Learning and evaluating a differentially private pre-trained language model},
  author={Hoory, Shlomo and Feder, Amir and Tendler, Avichai and Erell, Sofia and Peled-Cohen, Alon and Laish, Itay and Nakhost, Hootan and Stemmer, Uri and Benjamini, Ayelet and Hassidim, Avinatan and Matias, Yossi},
  booktitle={Findings of the Association for Computational Linguistics: EMNLP 2021},
  pages={1178--1189},
  year={2021}
}

@inproceedings{
li2022llmdp,
title={Large Language Models Can Be Strong Differentially Private Learners},
author={Xuechen Li and Florian Tramer and Percy Liang and Tatsunori Hashimoto},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=bVuP3ltATMz}
}

@article{mehta2022large,
  title={Large Scale Transfer Learning for Differentially Private Image Classification},
  author={Mehta, Harsh and Thakurta, Abhradeep and Kurakin, Alexey and Cutkosky, Ashok},
  journal={arXiv preprint arXiv:2205.02973},
  year={2022}
}

@article{gorbunov2020stochastic,
  title={Stochastic optimization with heavy-tailed noise via accelerated gradient clipping},
  author={Gorbunov, Eduard and Danilova, Marina and Gasnikov, Alexander},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15042--15053},
  year={2020}
}

@inproceedings{duchi2010composite,
  title={Composite Objective Mirror Descent.},
  author={Duchi, John C and Shalev-Shwartz, Shai and Singer, Yoram and Tewari, Ambuj},
  booktitle={COLT},
  volume={10},
  pages={14--26},
  year={2010},
  organization={Citeseer}
}

@article{vural2022mirror,
  title={Mirror Descent Strikes Again: Optimal Stochastic Convex Optimization under Infinite Noise Variance},
  author={Vural, Nuri Mert and Yu, Lu and Balasubramanian, Krishnakumar and Volgushev, Stanislav and Erdogdu, Murat A},
  journal={arXiv preprint arXiv:2202.11632},
  year={2022}
}

@article{ville1939etude,
  title={Etude critique de la notion de collectif, Gauthier-Villars, Paris, 1939},
  author={Ville, Jean},
  journal={Monographies des Probabilit{\'e}s. Calcul des Probabilit{\'e}s et ses Applications},
  year={1939}
}

@article{kulkarni2021private,
  title={Private non-smooth empirical risk minimization and stochastic convex optimization in subquadratic steps},
  author={Kulkarni, Janardhan and Lee, Yin Tat and Liu, Daogao},
  journal={arXiv preprint arXiv:2103.15352},
  year={2021}
}

@article{kamath2021improved,
  title={Improved rates for differentially private stochastic convex optimization with heavy-tailed data},
  author={Kamath, Gautam and Liu, Xingtu and Zhang, Huanyu},
  journal={arXiv preprint arXiv:2106.01336},
  year={2021}
}

@article{bassily2020stability,
  title={Stability of stochastic gradient descent on nonsmooth convex losses},
  author={Bassily, Raef and Feldman, Vitaly and Guzm{\'a}n, Crist{\'o}bal and Talwar, Kunal},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={4381--4391},
  year={2020}
}


@article{tropp2011freedman,
  title={Freedman's inequality for matrix martingales},
  author={Tropp, Joel},
  journal={Electronic Communications in Probability},
  volume={16},
  pages={262--270},
  year={2011},
  publisher={Institute of Mathematical Statistics and Bernoulli Society}
}

@inproceedings{mhammedi2019lipschitz,
  title={Lipschitz adaptivity with multiple learning rates in online learning},
  author={Mhammedi, Zakaria and Koolen, Wouter M and Van Erven, Tim},
  booktitle={Conference on Learning Theory},
  pages={2490--2511},
  year={2019},
  organization={PMLR}
}


@InProceedings{jacobsen2022parameter,
  title = 	 {Parameter-free Mirror Descent},
  author =       {Jacobsen, Andrew and Cutkosky, Ashok},
  booktitle = 	 {Proceedings of Thirty Fifth Conference on Learning Theory},
  pages = 	 {4160--4211},
  year = 	 {2022},
  editor = 	 {Loh, Po-Ling and Raginsky, Maxim},
  volume = 	 {178},
  series = 	 {Proceedings of Machine Learning Research},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v178/jacobsen22a/jacobsen22a.pdf},
  url = 	 {https://proceedings.mlr.press/v178/jacobsen22a.html},
  abstract = 	 {We develop a modified online mirror descent framework that is suitable for building adaptive and parameter-free algorithms in unbounded domains. We leverage this technique to develop the first unconstrained online linear optimization algorithm achieving an optimal dynamic regret bound, and we further demonstrate that natural strategies based on Follow-the-Regularized-Leader are unable to achieve similar results. We also apply our mirror descent framework to build new parameter-free implicit updates, as well as a simplified and improved unconstrained scale-free algorithm.}
}


@inproceedings{mhammedi2022efficient,
  title={Efficient projection-free online convex optimization with membership oracle},
  author={Mhammedi, Zakaria},
  booktitle={Conference on Learning Theory},
  pages={5314--5390},
  year={2022},
  organization={PMLR}
}

@article{mhammedi2022exploiting,
  title={Exploiting the Curvature of Feasible Sets for Faster Projection-Free Online Learning},
  author={Mhammedi, Zakaria},
  journal={arXiv preprint arXiv:2205.11470},
  year={2022}
}

@article{orabona2021tight,
  title={Tight concentrations and confidence sequences from the regret of universal portfolio},
  author={Orabona, Francesco and Jun, Kwang-Sung},
  journal={arXiv preprint arXiv:2110.14099},
  year={2021}
}

@article{waudby2020estimating,
  title={Estimating means of bounded random variables by betting},
  author={Waudby-Smith, Ian and Ramdas, Aaditya},
  journal={arXiv preprint arXiv:2010.09686},
  year={2020}
}

@article{mcmahan2010unified,
  title={A unified view of regularized dual averaging and mirror descent with implicit updates},
  author={McMahan, H Brendan},
  journal={arXiv preprint arXiv:1009.3240},
  year={2010}
}

@inproceedings{kulis2010implicit,
  title={Implicit online learning},
  author={Kulis, Brian and Bartlett, Peter L},
  booktitle={Proceedings of the 27th International Conference on Machine Learning (ICML-10)},
  pages={575--582},
  year={2010}
}

@inproceedings{li2014efficient,
  title={Efficient mini-batch training for stochastic optimization},
  author={Li, Mu and Zhang, Tong and Chen, Yuqiang and Smola, Alexander J},
  booktitle={Proceedings of the 20th ACM SIGKDD international conference on Knowledge discovery and data mining},
  pages={661--670},
  year={2014}
}
@inproceedings{wang2017memory,
  title={Memory and communication efficient distributed stochastic optimization with minibatch prox},
  author={Wang, Jialei and Wang, Weiran and Srebro, Nathan},
  booktitle={Conference on Learning Theory},
  pages={1882--1919},
  year={2017},
  organization={PMLR}
}

@article{parikh2014proximal,
  title={Proximal algorithms},
  author={Parikh, Neal and Boyd, Stephen},
  journal={Foundations and trends{\textregistered} in Optimization},
  volume={1},
  number={3},
  pages={127--239},
  year={2014},
  publisher={Now Publishers, Inc.}
}

@article{kivinen1997exponentiated,
  title={Exponentiated gradient versus gradient descent for linear predictors},
  author={Kivinen, Jyrki and Warmuth, Manfred K},
  journal={information and computation},
  volume={132},
  number={1},
  pages={1--63},
  year={1997},
  publisher={Elsevier}
}

@article{crammer2006online,
  title={Online Passive-Aggressive Algorithms},
  author={Crammer, Koby and Dekel, Ofer and Keshet, Joseph and Shalev-Shwartz, Shai and Singer, Yoram},
  journal={Journal of Machine Learning Research},
  volume={7},
  number={19},
  pages={551--585},
  year={2006}
}

@article{campolongo2020temporal,
  title={Temporal variability in implicit online learning},
  author={Campolongo, Nicolo and Orabona, Francesco},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={12377--12387},
  year={2020}
}

@article{campolongo2021closer,
  title={A closer look at temporal variability in dynamic online learning},
  author={Campolongo, Nicolo and Orabona, Francesco},
  journal={arXiv preprint arXiv:2102.07666},
  year={2021}
}

@article{levy2017online,
  title={Online to offline conversions, universality and adaptive minibatch sizes},
  author={Levy, Kfir},
  journal={Advances in Neural Information Processing Systems},
  volume={30},
  year={2017},
  pages={1613--1622},
}

article{levy2018online,
  title={Online adaptive methods, universality and acceleration},
  author={Levy, Kfir Y and Yurtsever, Alp and Cevher, Volkan},
  journal={Advances in Neural Information Processing Systems},
  volume={31},
  year={2018}
}

@article{van2015fast,
  title={Fast rates in statistical and online learning},
  author={Van Erven, Tim and Grunwald, Peter and Mehta, Nishant A and Reid, Mark and Williamson, Robert},
  year={2015},
  publisher={MIT Press}
}

@inproceedings{wei2018more,
  title={More adaptive algorithms for adversarial bandits},
  author={Wei, Chen-Yu and Luo, Haipeng},
  booktitle={Conference On Learning Theory},
  pages={1263--1291},
  year={2018},
  organization={PMLR}
}



@inproceedings{flaspohler2021online,
  title={Online learning with optimism and delay},
  author={Flaspohler, Genevieve E and Orabona, Francesco and Cohen, Judah and Mouatadid, Soukayna and Oprescu, Miruna and Orenstein, Paulo and Mackey, Lester},
  booktitle={International Conference on Machine Learning},
  pages={3363--3373},
  year={2021},
  organization={PMLR}
}



@article{radford2018improving,
 author = {Radford, Alec and Narasimhan, Karthik and Salimans, Tim and Sutskever, Ilya},
 publisher = {OpenAI},
 title = {Improving language understanding by generative pre-training},
 year = {2018}
}

@article{radford2019language,
 author = {Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
 journal = {OpenAI blog},
 number = {8},
 pages = {9},
 title = {Language models are unsupervised multitask learners},
 volume = {1},
 year = {2019}
}

@article{liu2019roberta,
 author = {Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
 journal = {ArXiv preprint},
 title = {Roberta: A robustly optimized bert pretraining approach},
 url = {https://arxiv.org/abs/1907.11692},
 volume = {abs/1907.11692},
 year = {2019}
}

@article{avsec2021effective,
 author = {Avsec, Ziga and Agarwal, Vikram and Visentin, Daniel and Ledsam, Joseph R and Grabska-Barwinska, Agnieszka and Taylor, Kyle R and Assael, Yannis and Jumper, John and Kohli, Pushmeet and Kelley, David R},
 journal = {Nature methods},
 number = {10},
 pages = {1196--1203},
 publisher = {Nature Publishing Group},
 title = {Effective gene expression prediction from sequence by integrating long-range interactions},
 volume = {18},
 year = {2021}
}

@article{alipanahi2015predicting,
  title={Predicting the sequence specificities of DNA-and RNA-binding proteins by deep learning},
  author={Alipanahi, Babak and Delong, Andrew and Weirauch, Matthew T and Frey, Brendan J},
  journal={Nature biotechnology},
  volume={33},
  number={8},
  pages={831--838},
  year={2015},
  publisher={Nature Publishing Group}
}

@article{zhou2015predicting,
  title={Predicting effects of noncoding variants with deep learning--based sequence model},
  author={Zhou, Jian and Troyanskaya, Olga G},
  journal={Nature methods},
  volume={12},
  number={10},
  pages={931--934},
  year={2015},
  publisher={Nature Publishing Group}
}





@article{jumper2021highly,
 author = {Jumper, John and Evans, Richard and Pritzel, Alexander and Green, Tim and Figurnov, Michael and Ronneberger, Olaf and Tunyasuvunakool, Kathryn and Bates, Russ and Zidek, Augustin and Potapenko, Anna and Bridgland, Alex and Meyer, Clemens and  Kohl, Simon A. A. and  Ballard, Andrew J. and  Cowie, Andrew and Romera-Paredes, Bernardino and Nikolov, Stanislav and Jain, Rishub and Adler, Jonas and Back, Trevor and Petersen, Stig and Reiman, David and Clancy, Ellen and Zielinski, Michal and Steinegger, Martin and Pacholska, Michalina and Berghammer, Tamas and Bodenstein, Sebastian and Silver, David and Vinyals, Oriol and Senior, Andrew W. and Kavukcuoglu, Koray and Kohli, Pushmeet and Hassabis, Demis},
 journal = {Nature},
 number = {7873},
 pages = {583--589},
 publisher = {Nature Publishing Group},
 title = {Highly accurate protein structure prediction with AlphaFold},
 volume = {596},
 year = {2021}
}


@inproceedings{devlin2018bert,
 address = {Minneapolis, Minnesota},
 author = {Devlin, Jacob  and
Chang, Ming-Wei  and
Lee, Kenton  and
Toutanova, Kristina},
 booktitle = {Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
 doi = {10.18653/v1/N19-1423},
 pages = {4171--4186},
 publisher = {Association for Computational Linguistics},
 title = {{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding},
 url = {https://aclanthology.org/N19-1423},
 year = {2019}
}


@inproceedings{vaswani2017attention,
 author = {Ashish Vaswani and
Noam Shazeer and
Niki Parmar and
Jakob Uszkoreit and
Llion Jones and
Aidan N. Gomez and
Lukasz Kaiser and
Illia Polosukhin},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17.bib},
 booktitle = {Advances in Neural Information Processing Systems 30: Annual Conference
on Neural Information Processing Systems 2017, December 4-9, 2017,
Long Beach, CA, {USA}},
 editor = {Isabelle Guyon and
Ulrike von Luxburg and
Samy Bengio and
Hanna M. Wallach and
Rob Fergus and
S. V. N. Vishwanathan and
Roman Garnett},
 pages = {5998--6008},
 timestamp = {Thu, 21 Jan 2021 00:00:00 +0100},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html},
 year = {2017}
}

@inproceedings{dosovitskiy2021image,
 author = {Alexey Dosovitskiy and
Lucas Beyer and
Alexander Kolesnikov and
Dirk Weissenborn and
Xiaohua Zhai and
Thomas Unterthiner and
Mostafa Dehghani and
Matthias Minderer and
Georg Heigold and
Sylvain Gelly and
Jakob Uszkoreit and
Neil Houlsby},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/iclr/DosovitskiyB0WZ21.bib},
 booktitle = {9th International Conference on Learning Representations, {ICLR} 2021,
Virtual Event, Austria, May 3-7, 2021},
 publisher = {OpenReview.net},
 timestamp = {Wed, 23 Jun 2021 01:00:00 +0200},
 title = {An Image is Worth 16x16 Words: Transformers for Image Recognition
at Scale},
 url = {https://openreview.net/forum?id=YicbFdNTTy},
 year = {2021}
}

@article{ba2016layer,
  title={Layer normalization},
  author={Ba, Jimmy Lei and Kiros, Jamie Ryan and Hinton, Geoffrey E},
  journal={arXiv preprint arXiv:1607.06450},
  year={2016}
}

@article{santurkar2018does,
  title={How does batch normalization help optimization?},
  author={Santurkar, Shibani and Tsipras, Dimitris and Ilyas, Andrew and Madry, Aleksander},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}

@inproceedings{ioffe2015batch,
  title={Batch normalization: Accelerating deep network training by reducing internal covariate shift},
  author={Ioffe, Sergey and Szegedy, Christian},
  booktitle={International conference on machine learning},
  pages={448--456},
  year={2015},
  organization={PMLR}
}

@article{veit2016residual,
  title={Residual networks behave like ensembles of relatively shallow networks},
  author={Veit, Andreas and Wilber, Michael J and Belongie, Serge},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@inproceedings{xiong2020layer,
  title={On layer normalization in the transformer architecture},
  author={Xiong, Ruibin and Yang, Yunchang and He, Di and Zheng, Kai and Zheng, Shuxin and Xing, Chen and Zhang, Huishuai and Lan, Yanyan and Wang, Liwei and Liu, Tieyan},
  booktitle={International Conference on Machine Learning},
  pages={10524--10533},
  year={2020},
  organization={PMLR}
}

@article{nguyen2019transformers,
  title={Transformers without tears: Improving the normalization of self-attention},
  author={Nguyen, Toan Q and Salazar, Julian},
  journal={arXiv preprint arXiv:1910.05895},
  year={2019}
}

@article{goyal2017accurate,
  title={Accurate, large minibatch sgd: Training imagenet in 1 hour},
  author={Goyal, Priya and Doll{\'a}r, Piotr and Girshick, Ross and Noordhuis, Pieter and Wesolowski, Lukasz and Kyrola, Aapo and Tulloch, Andrew and Jia, Yangqing and He, Kaiming},
  journal={arXiv preprint arXiv:1706.02677},
  year={2017}
}

@article{li2020demystifying,
  title = {Demystifying GPT 3},
  author = {Chuan Li},
  journal={LambdaLabs Blog},
  year = {2020},
  url={https://lambdalabs.com/blog/demystifying-gpt-3/},
  note = {Accessed: 2022-18-7}
}

@article{knight2021ai,
  title = {AI’s Smarts Now Come With a Big Price Tag},
  author = {Will Knight},
  journal={Wired},
  year = {2021},
  url={https://www.wired.com/story/ai-smarts-big-price-tag/},
  note = {Accessed: 2022-18-7}
}


@article{zhang2022democratizing,
  title = {Democratizing access to large-scale language models with OPT-175B},
  author = {Susan Zhang, Mona Diab, Luke Zettlemoyer},
  journal={Meta AI Blog},
  year = {2022},
  url={https://ai.facebook.com/blog/democratizing-access-to-large-scale-language-models-with-opt-175b/},
  note = {Accessed: 2022-18-7}
}

@inproceedings{wang2020adaptivity,
  title={Adaptivity and optimality: A universal algorithm for online convex optimization},
  author={Wang, Guanghui and Lu, Shiyin and Zhang, Lijun},
  booktitle={Uncertainty in Artificial Intelligence},
  pages={659--668},
  year={2020},
  organization={PMLR}
}


@article{chowdhery2022palm,
  title={PaLM: Scaling Language Modeling with Pathways},
  author={Aakanksha Chowdhery and Sharan Narang and Jacob Devlin and Maarten Bosma and Gaurav Mishra and Adam Roberts and Paul Barham and Hyung Won Chung and Charles Sutton and Sebastian Gehrmann and Parker Schuh and Kensen Shi and Sasha Tsvyashchenko and Joshua Maynez and Abhishek B Rao and Parker Barnes and Yi Tay and Noam M. Shazeer and Vinodkumar Prabhakaran and Emily Reif and Nan Du and Benton C. Hutchinson and Reiner Pope and James Bradbury and Jacob Austin and Michael Isard and Guy Gur-Ari and Pengcheng Yin and Toju Duke and Anselm Levskaya and Sanjay Ghemawat and Sunipa Dev and Henryk Michalewski and Xavier Garc{\'i}a and Vedant Misra and Kevin Robinson and Liam Fedus and Denny Zhou and Daphne Ippolito and David Luan and Hyeontaek Lim and Barret Zoph and Alexander Spiridonov and Ryan Sepassi and David Dohan and Shivani Agrawal and Mark Omernick and Andrew M. Dai and Thanumalayan Sankaranarayana Pillai and Marie Pellat and Aitor Lewkowycz and Erica Oliveira Moreira and Rewon Child and Oleksandr Polozov and Katherine Lee and Zongwei Zhou and Xuezhi Wang and Brennan Saeta and Mark D{\'i}az and Orhan Firat and Michele Catasta and Jason Wei and Kathleen S. Meier-Hellstern and Douglas Eck and Jeff Dean and Slav Petrov and Noah Fiedel},
  journal={ArXiv},
  year={2022},
  volume={abs/2204.02311}
}


@article{zhang2022opt,
  title={OPT: Open Pre-trained Transformer Language Models},
  author={Susan Zhang and Stephen Roller and Naman Goyal and Mikel Artetxe and Moya Chen and Shuohui Chen and Christopher Dewan and Mona Diab and Xian Li and Xi Victoria Lin and Todor Mihaylov and Myle Ott and Sam Shleifer and Kurt Shuster and Daniel Simig and Punit Singh Koura and Anjali Sridhar and Tianlu Wang and Luke Zettlemoyer},
  journal={ArXiv},
  year={2022},
  volume={abs/2205.01068}
}


@article{smith2018disciplined,
  title={A disciplined approach to neural network hyper-parameters: Part 1 learning rate, batch size, momentum, and weight decay},
  author={Smith, Leslie N},
  journal={arXiv preprint arXiv:1803.09820},
  year={2018}
}


@article{patterson2021carbon,
  title={Carbon emissions and large neural network training},
  author={Patterson, David and Gonzalez, Joseph and Le, Quoc and Liang, Chen and Munguia, Lluis-Miquel and Rothchild, Daniel and So, David and Texier, Maud and Dean, Jeff},
  journal={arXiv preprint arXiv:2104.10350},
  year={2021}
}

@inproceedings{bartlett2007adaptive,
  title={Adaptive online gradient descent},
  author={Bartlett, Peter and Hazan, Elad and Rakhlin, Alexander},
  booktitle={Advances in Neural Information Processing Systems 20: Proceedings of the 2007 Conference},
  pages={65--72},
  year={2009},
  organization={Neural Information Processing Systems (NIPS) Foundation}
}

@inproceedings{zhang2004solving,
  title={Solving large scale linear prediction problems using stochastic gradient descent algorithms},
  author={Zhang, Tong},
  booktitle={Proceedings of the twenty-first international conference on Machine learning},
  pages={116},
  year={2004}
}


@article{stich2020error,
  title={The error-feedback framework: Better rates for sgd with delayed gradients and compressed updates},
  author={Stich, Sebastian U and Karimireddy, Sai Praneeth},
  journal={Journal of Machine Learning Research},
  volume={21},
  pages={1--36},
  year={2020}
}
@article{stich2018local,
  title={Local SGD converges fast and communicates little},
  author={Stich, Sebastian U},
  journal={arXiv preprint arXiv:1805.09767},
  year={2018}
}

@inproceedings{arjevani2020tight,
  title={A tight convergence analysis for stochastic gradient descent with delayed updates},
  author={Arjevani, Yossi and Shamir, Ohad and Srebro, Nathan},
  booktitle={Algorithmic Learning Theory},
  pages={111--132},
  year={2020},
  organization={PMLR}
}
@article{mcmahan2014delay,
  title={Delay-tolerant algorithms for asynchronous distributed online learning},
  author={McMahan, Brendan and Streeter, Matthew},
  journal={Advances in Neural Information Processing Systems},
  volume={27},
  year={2014}
}

@inproceedings{joulani2016delay,
  title={Delay-tolerant online convex optimization: Unified analysis and adaptive-gradient algorithms},
  author={Joulani, Pooria and Gyorgy, Andras and Szepesv{\'a}ri, Csaba},
  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
  volume={30},
  number={1},
  year={2016}
}


@inproceedings{flaspohler2021online,
  title={Online learning with optimism and delay},
  author={Flaspohler, Genevieve E and Orabona, Francesco and Cohen, Judah and Mouatadid, Soukayna and Oprescu, Miruna and Orenstein, Paulo and Mackey, Lester},
  booktitle={International Conference on Machine Learning},
  pages={3363--3373},
  year={2021},
  organization={PMLR}
}

@inproceedings{draxler2018essentially,
  title={Essentially no barriers in neural network energy landscape},
  author={Draxler, Felix and Veschgini, Kambis and Salmhofer, Manfred and Hamprecht, Fred},
  booktitle={International conference on machine learning},
  pages={1309--1318},
  year={2018},
  organization={PMLR}
}


@article{garipov2018loss,
  title={Loss surfaces, mode connectivity, and fast ensembling of dnns},
  author={Garipov, Timur and Izmailov, Pavel and Podoprikhin, Dmitrii and Vetrov, Dmitry P and Wilson, Andrew G},
  journal={Advances in neural information processing systems},
  volume={31},
  pages={8803--8812},
  year={2018}
}


@article{he2019asymmetric,
  title={Asymmetric valleys: Beyond sharp and flat local minima},
  author={He, Haowei and Huang, Gao and Yuan, Yang},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@inproceedings{nguyen2019connected,
  title={On connected sublevel sets in deep learning},
  author={Nguyen, Quynh},
  booktitle={International conference on machine learning},
  pages={4790--4799},
  year={2019},
  organization={PMLR}
}

@inproceedings{gotmare2018closer,
  title={A Closer Look at Deep Learning Heuristics: Learning rate restarts, Warmup and Distillation},
  author={Gotmare, Akhilesh and Keskar, Nitish Shirish and Xiong, Caiming and Socher, Richard},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{kavis2019unixgrad,
  title={UniXGrad: A Universal, Adaptive Algorithm with Optimal Guarantees for Constrained Optimization.},
  author={Kavis, Ali and Levy, Kfir Y and Bach, Francis and Cevher, Volkan},
  booktitle={NeurIPS},
  pages={6257--6266},
  year={2019}
}

@inproceedings{bach2019universal,
  title={A universal algorithm for variational inequalities adaptive to smoothness and noise},
  author={Bach, Francis and Levy, Kfir Y},
  booktitle={Conference on learning theory},
  pages={164--194},
  year={2019},
  organization={PMLR}
}


@article{OpenML2013,
  author = {Joaquin Vanschoren and Jan N. van Rijn and Bernd Bischl and Luis Torgo},
  title = {OpenML: networked science in machine learning},
  journal = {SIGKDD Explorations},
  volume = {15},
  number = {2},
  year = {2013},
  pages = {49-60},
  url = {http://doi.acm.org/10.1145/2641190.264119},
  doi = {10.1145/2641190.2641198},
  publisher = {ACM}
}

@inproceedings{he2015delving,
  title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE international conference on computer vision},
  pages={1026--1034},
  year={2015}
}

@misc{paramfreetutorial,
  title = {ICML 2020 Tutorial: Parameter-Free Online Optimization},
  author={Orabona, Francesco and Cutkosky, Ashok},
  howpublished = {\url{https://parameterfree.com/icml-tutorial/}}
}


@InProceedings{faw22power,
  title = 	 {The Power of Adaptivity in SGD: Self-Tuning Step Sizes with Unbounded Gradients and Affine Variance},
  author =       {Faw, Matthew and Tziotis, Isidoros and Caramanis, Constantine and Mokhtari, Aryan and Shakkottai, Sanjay and Ward, Rachel},
  booktitle = 	 {Proceedings of Thirty Fifth Conference on Learning Theory},
  pages = 	 {313--355},
  year = 	 {2022}
}


@article{khaled2020better,
  title={Better theory for SGD in the nonconvex world},
  author={Khaled, Ahmed and Richt{\'a}rik, Peter},
  journal={arXiv preprint arXiv:2002.03329},
  year={2020}
}

@inproceedings{li2021second,
  title={A second look at exponential and cosine step sizes: Simplicity, adaptivity, and performance},
  author={Li, Xiaoyu and Zhuang, Zhenxun and Orabona, Francesco},
  booktitle={International Conference on Machine Learning},
  pages={6553--6564},
  year={2021},
  organization={PMLR}
}

@inproceedings{karimi2016linear,
  title={Linear convergence of gradient and proximal-gradient methods under the polyak-{\l}ojasiewicz condition},
  author={Karimi, Hamed and Nutini, Julie and Schmidt, Mark},
  booktitle={Joint European conference on machine learning and knowledge discovery in databases},
  pages={795--811},
  year={2016},
  organization={Springer}
}

@article{davis2020stochastic,
  title={Stochastic subgradient method converges on tame functions},
  author={Davis, Damek and Drusvyatskiy, Dmitriy and Kakade, Sham and Lee, Jason D},
  journal={Foundations of computational mathematics},
  volume={20},
  number={1},
  pages={119--154},
  year={2020},
  publisher={Springer}
}

@article{sanborn2015chromatin,
  title={Chromatin extrusion explains key features of loop and domain formation in wild-type and engineered genomes},
  author={Sanborn, Adrian L and Rao, Suhas SP and Huang, Su-Chen and Durand, Neva C and Huntley, Miriam H and Jewett, Andrew I and Bochkov, Ivan D and Chinnappan, Dharmaraj and Cutkosky, Ashok and Li, Jian and Geeting, Kristopher P and Gnirke, Andreas and Melnikov, Alexandre and McKenna, Doug and tamenova, Elena K S and Lander, Eric S and Aiden, Erez Lieberman },
  journal={Proceedings of the National Academy of Sciences},
  volume={112},
  number={47},
  pages={E6456--E6465},
  year={2015},
  publisher={National Acad Sciences}
}

@article{mehta2022long,
  title={Long Range Language Modeling via Gated State Spaces},
  author={Mehta, Harsh and Gupta, Ankit and Cutkosky, Ashok and Neyshabur, Behnam},
  journal={arXiv preprint arXiv:2206.13947},
  year={2022}
}


@inproceedings{mehta2020extreme,
  title={Extreme Memorization via Scale of Initialization},
  author={Mehta, Harsh and Cutkosky, Ashok and Neyshabur, Behnam},
  booktitle={International Conference on Learning Representations},
  year={2020}
}

@inproceedings{shazeer2018adafactor,
  title={Adafactor: Adaptive learning rates with sublinear memory cost},
  author={Shazeer, Noam and Stern, Mitchell},
  booktitle={International Conference on Machine Learning},
  pages={4596--4604},
  year={2018},
  organization={PMLR}
}


@article{bernstein2020distance,
  title={On the distance between two neural networks and the stability of learning},
  author={Bernstein, Jeremy and Vahdat, Arash and Yue, Yisong and Liu, Ming-Yu},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={21370--21381},
  year={2020}
}

@article{zhang2020adaptive,
  title={Why are adaptive methods good for attention models?},
  author={Zhang, Jingzhao and Karimireddy, Sai Praneeth and Veit, Andreas and Kim, Seungyeon and Reddi, Sashank and Kumar, Sanjiv and Sra, Suvrit},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15383--15393},
  year={2020}
}

@article{zhang2020improved,
  title={Improved analysis of clipping algorithms for non-convex optimization},
  author={Zhang, Bohang and Jin, Jikai and Fang, Cong and Wang, Liwei},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={15511--15521},
  year={2020}
}

@inproceedings{mai2021stability,
  title={Stability and convergence of stochastic gradient clipping: Beyond lipschitz continuity and smoothness},
  author={Mai, Vien V and Johansson, Mikael},
  booktitle={International Conference on Machine Learning},
  pages={7325--7335},
  year={2021},
  organization={PMLR}
}

@article{zhou2020towards,
  title={Towards theoretically understanding why sgd generalizes better than adam in deep learning},
  author={Zhou, Pan and Feng, Jiashi and Ma, Chao and Xiong, Caiming and Hoi, Steven Chu Hong and others},
  journal={Advances in Neural Information Processing Systems},
  volume={33},
  pages={21285--21296},
  year={2020}
}
@article{kavis2022high,
  title={High Probability Bounds for a Class of Nonconvex Algorithms with AdaGrad Stepsize},
  author={Kavis, Ali and Levy, Kfir Yehuda and Cevher, Volkan},
  journal={arXiv preprint arXiv:2204.02833},
  year={2022}
}

@article{madden2020high,
  title={High probability convergence bounds for stochastic gradient descent assuming the polyak-lojasiewicz inequality},
  author={Madden, Liam and Dall'Anese, Emiliano and Becker, Stephen},
  journal={arXiv preprint arXiv:2006.05610},
  year={2020}
}

@misc{liam2020high,
  doi = {10.48550/ARXIV.2006.05610},
  url = {https://arxiv.org/abs/2006.05610},
  author = {Madden, Liam and Dall'Anese, Emiliano and Becker, Stephen},
  title = {High-probability Convergence Bounds for Non-convex Stochastic Gradient Descent},
  publisher = {arXiv},
  year = {2020},
  copyright = {arXiv.org perpetual, non-exclusive license}
}

@article{harvey2019simple,
  title={Simple and optimal high-probability bounds for strongly-convex stochastic gradient descent},
  author={Harvey, Nicholas JA and Liaw, Christopher and Randhawa, Sikander},
  journal={arXiv preprint arXiv:1909.00843},
  year={2019}
}

@inproceedings{li2022high,
  title={High Probability Guarantees for Nonconvex Stochastic Gradient Descent with Heavy Tails},
  author={Li, Shaojie and Liu, Yong},
  booktitle={International Conference on Machine Learning},
  pages={12931--12963},
  year={2022},
  organization={PMLR}
}

@article{carmon2022making,
  title={Making SGD Parameter-Free},
  author={Carmon, Yair and Hinder, Oliver},
  journal={Conference on Learning Theory},
  year={2022}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}