Merge /next of 0.15

.gitignore

@@ -8,3 +8,7 @@ training/tf/venv
 leelaz-model*
 *.orig
 leelaz_opencl_tuning
+/build-autogtp-*
+/build-validation-*
+.vs/
+build/

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "gtest"]
 	path = gtest
 	url = https://github.com/google/googletest.git
+[submodule "src/Eigen"]
+	path = src/Eigen
+	url = https://github.com/eigenteam/eigen-git-mirror

.travis.yml

@@ -15,14 +15,20 @@ jobs:
       - docker build -f Dockerfiles/Dockerfile.gpu -t leela-zero:gpu .
       - docker run leela-zero:gpu
     - script:
-      - docker build -f Dockerfiles/Dockerfile.gpu-half -t leela-zero:gpu-half .
-      - docker run leela-zero:gpu-half
+      - docker build -f Dockerfiles/Dockerfile.gpu-blas -t leela-zero:gpu-blas .
+      - docker run leela-zero:gpu-blas
     - script:
       - docker build -f Dockerfiles/Dockerfile.cpu -t leela-zero:cpu .
       - docker run leela-zero:cpu
+    - script:
+      - docker build -f Dockerfiles/Dockerfile.cpu-blas -t leela-zero:cpu-blas .
+      - docker run leela-zero:cpu-blas
     - script:
       - docker build -f Dockerfiles/Dockerfile.tests -t leela-zero:tests .
       - docker run leela-zero:tests
+    - script:
+      - docker build -f Dockerfiles/Dockerfile.tests-blas -t leela-zero:tests-blas .
+      - docker run leela-zero:tests-blas
     - stage: style
       before_install:
-      script: find . -regex ".*\.\(cpp\|h\|hpp\)" -not -regex ".*moc_.*.cpp" -not -path "./gtest/*" -not -path "./training/*" -not -path "./src/half/*" -not -path "./src/CL/*" | xargs python2 scripts/cpplint.py --filter=-build/c++11,-build/include,-build/include_order,-build/include_what_you_use,-build/namespaces,-readability/braces,-readability/casting,-readability/fn_size,-readability/namespace,-readability/todo,-runtime/explicit,-runtime/indentation_namespace,-runtime/int,-runtime/references,-whitespace/blank_line,-whitespace/braces,-whitespace/comma,-whitespace/comments,-whitespace/empty_loop_body,-whitespace/line_length,-whitespace/semicolon
+      script: find . -regex ".*\.\(cpp\|h\|hpp\)" -not -regex ".*moc_.*.cpp" -not -path "./gtest/*" -not -path "./training/*" -not -path "./src/half/*" -not -path "./src/CL/*" -not -path "./src/Eigen/*" | xargs python2 scripts/cpplint.py --filter=-build/c++11,-build/include,-build/include_order,-build/include_what_you_use,-build/namespaces,-readability/braces,-readability/casting,-readability/fn_size,-readability/namespace,-readability/todo,-runtime/explicit,-runtime/indentation_namespace,-runtime/int,-runtime/references,-whitespace/blank_line,-whitespace/braces,-whitespace/comma,-whitespace/comments,-whitespace/empty_loop_body,-whitespace/line_length,-whitespace/semicolon

AUTHORS

@@ -1,40 +1,63 @@
 Gian-Carlo Pascutto
 Seth Troisi
-Marco Calignano
 Henrik Forstén
-Andy Olsen
-Junhee Yoo
 TFiFiE
+Junhee Yoo
+Marco Calignano
+Andy Olsen
+Hersmunch
 Bood Qian
 Peter Wen
 ywrt
 Arseny Krasutsky
 earthengine
+Jonathan Roy
+Mankit Pong
 michael
-Hersmunch
 Barry G Becker
+Junyan Xu
 Maks Kolman
+kuba97531
 Antti Korhonen
-Mankit Pong
+Chin-Chang Yang
 Xingcan LAN
+bittsitt
 tux3
 5525345551
+Adrian Petrescu
+Akita Noek
 Alderi-Tokori
+Alexander Taylor
+Ancalagon
 Ashley Griffiths
+Barry Becker
 Ed Lee
 Eddh
 F. Huizinga
 FFLaguna
 Jiannan Liu
 Joe Ren
-Jonathan Roy
+LL145
+Mark Andrew Gerads
 Nate
+OmnipotentEntity
+Przemek Wesołek
+Sebastian H
 Shen-Ta Hsieh(BestSteve)
 Virgile Andreani
+Ximin Luo
+ZenStone
+Zhenzhen Zhan
 afalturki
+betterworld
 cheshirecats
+dbosst
 fohristiwhirl
 gaieepo
+ncaq
 tterava
+wonderingabout
+zediir
+zliu1022
 Пахотин Иван
 Google LLC

CMakeLists.txt

@@ -1,6 +1,6 @@
 # This file is part of Leela Zero.
 # Copyright (C) 2017 Marco Calignano
-# Copyright (C) 2017-2018 Gian-Carlo Pascutto and contributors
+# Copyright (C) 2017-2019 Gian-Carlo Pascutto and contributors
 # Leela Zero is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -15,14 +15,15 @@
 cmake_minimum_required(VERSION 3.1)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+include(GNUInstallDirs)
 
 project(leelaz)
 add_subdirectory(gtest EXCLUDE_FROM_ALL) # We don't want to install gtest, exclude it from `all`
 
 # Required Packages
 set(Boost_MIN_VERSION "1.58.0")
 set(Boost_USE_MULTITHREADED ON)
-find_package(Boost 1.58.0 REQUIRED program_options)
+find_package(Boost 1.58.0 REQUIRED program_options filesystem)
 find_package(Threads REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(OpenCL REQUIRED)
@@ -32,14 +33,20 @@ find_package(OpenCL REQUIRED)
 if(NOT APPLE)
   set(BLA_VENDOR OpenBLAS)
 endif()
-find_package(BLAS REQUIRED)
-find_path(BLAS_INCLUDE_DIRS openblas_config.h
-  /usr/include
-  /usr/local/include
-  /usr/include/openblas
-  /opt/OpenBLAS/include
-  /usr/include/x86_64-linux-gnu
-  $ENV{BLAS_HOME}/include)
+if(USE_BLAS)
+  message(STATUS "Looking for system BLAS/OpenBLAS library.")
+  find_package(BLAS REQUIRED)
+  find_path(BLAS_INCLUDE_DIRS openblas_config.h
+    /usr/include
+    /usr/local/include
+    /usr/include/openblas
+    /opt/OpenBLAS/include
+    /usr/include/x86_64-linux-gnu
+    $ENV{BLAS_HOME}/include)
+    add_definitions(-DUSE_BLAS)
+else()
+message(STATUS "Using built-in matrix library.")
+endif()
 find_package(Qt5Core)
 
 set(CMAKE_CXX_STANDARD 14)
@@ -79,7 +86,7 @@ if(USE_HALF)
   add_definitions(-DUSE_HALF)
 endif()
 
-set(IncludePath "${CMAKE_CURRENT_SOURCE_DIR}/src")
+set(IncludePath "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_SOURCE_DIR}/src/Eigen")
 set(SrcPath "${CMAKE_CURRENT_SOURCE_DIR}/src")
 
 include_directories(${IncludePath})
@@ -108,7 +115,7 @@ target_link_libraries(leelaz ${BLAS_LIBRARIES})
 target_link_libraries(leelaz ${OpenCL_LIBRARIES})
 target_link_libraries(leelaz ${ZLIB_LIBRARIES})
 target_link_libraries(leelaz ${CMAKE_THREAD_LIBS_INIT})
-install(TARGETS leelaz DESTINATION bin)
+install(TARGETS leelaz DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 if(Qt5Core_FOUND)
     if(NOT Qt5Core_VERSION VERSION_LESS "5.3.0")
@@ -135,3 +142,21 @@ target_link_libraries(tests ${BLAS_LIBRARIES})
 target_link_libraries(tests ${OpenCL_LIBRARIES})
 target_link_libraries(tests ${ZLIB_LIBRARIES})
 target_link_libraries(tests gtest_main ${CMAKE_THREAD_LIBS_INIT})
+
+include(GetGitRevisionDescription)
+git_describe(VERSION --tags)
+string(REGEX REPLACE "^v([0-9]+)\\..*" "\\1" MAJOR_VERSION "${VERSION}")
+string(REGEX REPLACE "^v[0-9]+\\.([0-9]+).*" "\\1" MINOR_VERSION "${VERSION}")
+
+SET(CPACK_GENERATOR "DEB")
+SET(CPACK_DEBIAN_PACKAGE_NAME "leelaz")
+SET(CPACK_DEBIAN_PACKAGE_VERSION "${MAJOR_VERSION}.${MINOR_VERSION}")
+SET(CPACK_DEBIAN_ARCHITECTURE ${CMAKE_SYSTEM_PROCESSOR})
+SET(CPACK_DEBIAN_PACKAGE_MAINTAINER "Gian-Carlo Pascutto https://github.com/gcp/leela-zero")
+SET(CPACK_DEBIAN_PACKAGE_DESCRIPTION "Go engine with no human-provided knowledge, modeled after the AlphaGo Zero paper.")
+SET(CPACK_DEBIAN_PACKAGE_PRIORITY "optional")
+SET(CPACK_DEBIAN_PACKAGE_SECTION "games")
+SET(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+SET(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${MAJOR_VERSION}.${MINOR_VERSION}")
+
+INCLUDE(CPack)

CONTRIBUTING.md

@@ -0,0 +1,110 @@
+# Contributing to Leela Zero
+
+## C++ Usage
+
+Leela Zero is written in C++14, and generally encourages writing in modern C++ style.
+
+This means that:
+
+* The code overwhelmingly uses Almost Always Auto style, and so should you.
+* Prefer range based for and non-member (c)begin/(c)end.
+* You can rely on boost 1.58.0 or later being present.
+* Manipulation of raw pointers is to be avoided as much as possible.
+* Prefer constexpr over defines or constants.
+* Prefer "using" over typedefs.
+* Prefer uniform initialization.
+* Prefer default initializers for member variables.
+* Prefer emplace_back and making use of move assignment.
+* Aim for const-correctness. Prefer passing non-trivial parameters by const reference.
+* Use header include guards, not #pragma once (pragma once is non-standard, has issues with detecting identical files, and is slower https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58770)
+* config.h is always the first file included.
+* Feel free to use templates, but remember that debugging obscure template metaprogramming bugs is not something people enjoy doing in their spare time.
+* Using exceptions is allowed.
+
+## Code Style
+
+* Look at the surrounding code and the rest of the project!
+* Indentation is 4 spaces. No tabs.
+* public/private/protected access modifiers are de-indented
+* Maximum line length is 80 characters. There are rare exceptions in the code, usually involving user-visible text strings.
+* Ifs are always braced, with very rare exceptions when everything fits on one line and doing it properly makes the code less readable.
+* The code generally avoids any pointer passing and allows non-const references for parameters. Still, for new code it should be preferred to a) put input parameters first b) use return values over output parameters.
+* Function arguments that wrap are aligned.
+* Member variables in a class have an m_ prefix and are private. Members of POD structs don't and aren't.
+* Constants and enum values are ALL_CAPS.
+* Variables are lowercase.
+* Function names are underscore_case.
+* Classes are CamelCase.
+* Comments are preferably full sentences with proper capitalization and a period.
+* Split the includes list into config.h, standard headers and our headers.
+
+If something is not addressed here or there is no similar code, the Google C++ Style Guide is always a good reference.
+
+We might move to enforce clang-format at some point.
+
+## Adding dependencies
+
+C++ does not quite have the package systems JavaScript and Rust have, so some restraint should be excercised when adding dependencies. Dependencies typically complicate the build for new contributors, especially on Windows, and reliance on specific, new versions can be a nuisance on Unix based systems.
+
+The restraints on modern header-only libraries are significantly less because they avoid most of the above problems.
+
+If a library is not mature and well-supported on Windows, Linux *and* macOS, you do not want it.
+
+This is not an excuse to re-invent the wheel.
+
+## Upgrading dependencies
+
+The code and dependencies should target the latest stable versions of Visual Studio/MSVC, and the latest stable/LTS releases of common Linux distros, with some additional delay as not everyone will be able to upgrade to a new stable/LTS right away.
+
+For example, upgrading to C++17 or boost 1.62.0 (oldest version in a Debian stable or Ubuntu LTS release) can be considered if there's a compelling use case and/or we can confirm it is supported on all platforms we reasonably target.
+
+## Merging contributions
+
+Contributions come in the form of pull requests against the "next" branch.
+
+They are rebased or squashed on top of the next branch, so the history will stay linear, i.e. no merge commits.
+
+Commit messages follow Linux kernel style: a summary phrase that is no more than 70-75 characters (but preferably <50) and describes both what the patch changes, as well as why the patch might be necessary.
+
+If the patch is to a specific subsystem (AutoGTP, Validation, ...) then prefix the summary by that subsystem (e.g. AutoGTP: ...).
+
+This is followed by a blank line, and a description that is wrapped at 72 characters. Good patch descriptions can be large time savers when someone has to bugfix the code afterwards.
+
+The end of the commit message should mention which (github) issue the patch fixes, if any, and the pull request it belongs to.
+
+Patches need to be reviewed before merging. Try to find the person who worked on the code last, or who has done work in nearby code (git blame is your friend, and this is why we write proper commit messages...). With some luck that is someone with write access to the repository. If not, you'll have to ping someone who does.
+
+Experience says that the majority of the pull requests won't live up to this ideal, which means that maintainers will have to squash patch series and clean up the commit message to be coherent before merging.
+
+If you are a person with write access to the repo, and are about to merge a commit, ask yourself the following question: am I confident enough that I understand this code, so that I can and am willing to go in and fix it if it turns out to be necessary? If the answer to this question is no, then do not merge the code. Not merging a contribution (quickly) is annoying for the individual contributor. Merging a bad contribution is annoying for everyone who wants to contribute now and in the future.
+
+If a contributor can't be bothered to fix up the trailing whitespace in their patch, odds are they aren't going to be willing to fix the threading bug it introduces either.
+
+## "Improvements" and Automagic
+
+Improvements to the engine that can affect strength should include supporting data. This means no-regression tests for functional changes, and a proof of strength improvement for things which are supposed to increase strength.
+
+The tools in the validation directory are well-fit for this purpose, as
+is the python tool "ringmaster".
+
+The number of configurable options should be limited where possible. If it is not possible for the author to make rules of thumb for suitable values for those options, then the majority of users have no hope of getting them right, and may mistakenly make the engine weaker. If you must introduce new ones, consider limiting their exposure to developers only via USE_TUNER and set a good default for them.
+
+## GTP Extensions
+
+GTP makes it possible to connect arbitrary engines to arbitrary interfaces.
+
+Unfortunately GTP 2 isn't extensive enough to realistically fit all needs of analysis GUIs, which means we have had to extend it. The lack of standardization here means that Go software is continously catching up to the chess world, especially after UCI was introduced. We should aim to make this situation better, not worse.
+
+This means that extensions have the possibility of outliving Leela Zero (or any GUIs) provided they are well thought out.
+
+It makes sense to be thoughtful here, consider the responsibilities of both GUI and engine, and try to come up with flexible building blocks rather than a plethora of commands for very specific use cases.
+
+Experience and previous discussions can help understanding:
+
+* lz-analyze "avoid" and "allow" were added in pull request [#1949](https://github.com/leela-zero/leela-zero/pull/1949).
+* lz-analyze got a side-to-move option in pull request [#1872](https://github.com/leela-zero/leela-zero/pull/1872) and [#1642](https://github.com/leela-zero/leela-zero/pull/1642).
+* lz-analyze got a "prior" tag in pull request [#1836](https://github.com/leela-zero/leela-zero/pull/1836).
+* lz-analyze was added in pull request [#1388](https://github.com/leela-zero/leela-zero/pull/1388).
+* lz-setoption was added in pull request [#1741](https://github.com/leela-zero/leela-zero/pull/1741).
+* Pull request [#2170](https://github.com/leela-zero/leela-zero/pull/2170) has some discussion regarding how to navigate SGF
+  files that were parsed by the engine via GTP.

Dockerfiles/Dockerfile.gpu-half → Dockerfiles/Dockerfile.cpu-blas

@@ -1,6 +1,6 @@
 FROM leela-zero:base
 
-# GPU 16-bit (memory storage) build
-RUN CXX=g++ CC=gcc cmake -DUSE_HALF=1 ..
+# CPU build
+RUN CXX=g++ CC=gcc cmake -DUSE_CPU_ONLY=1 -DUSE_BLAS=1 ..
 
 CMD cmake --build . --target leelaz --config Release -- -j2

Dockerfiles/Dockerfile.gpu-blas

@@ -0,0 +1,6 @@
+FROM leela-zero:base
+
+# GPU build
+RUN CXX=g++ CC=gcc cmake -DUSE_BLAS=1 ..
+
+CMD cmake --build . --target leelaz --config Release -- -j2

Dockerfiles/Dockerfile.tests-blas

@@ -0,0 +1,8 @@
+FROM leela-zero:base
+
+# CPU build
+RUN CXX=g++ CC=gcc cmake -DUSE_CPU_ONLY=1 -DUSE_BLAS=1 ..
+RUN cmake --build . --target tests --config Release -- -j2
+
+CMD ./tests
+