feature: TOASTER_WEBMAIL_PROXY=nginx (#595)

- feat: as an alternative to haproxy, use the installed nginx as the
reverse proxy
  - nginx: support TOASTER_WEBMAIL_PROXY=nginx
  - webmail: add https proxy config, for TOASTER_WEBMAIL_PROXY=nginx
- previously, haproxy sent proxy_protocol v2. Eventually, everything
should use Forwarded (RFC 7239), for now we use the lowest common
denominator: X-Forwarded-For
- fix: newsyslog.conf.d files must end with .conf
- fix(mail_dmarc): install .ini file in correct location
- data: standardize dirs in /data
  - mt: stage_setup_tls populates /data/etc/tls
  - dovecot: /data/etc/ssl -> /data/etc/tls
  - haproxy: /data/ssl.d -> /data/etc/tls.d
  - webmail: acme document root /data -> /data/htdocs
- letsencrypt: install acme.sh from ports
- letsencrypt: updates for etc/tls, add webmail deploy script
- mua: for testing, find or install curl, fixes #594
- mysql: use jail_is_running (noop)
- roundcube: configure SA user prefs plugin
- mt: added stage_enable_newsyslog, used in nginx, php, haraka, mongodb,
redis, rspamd, spamassassin
- geoip: switch updater to geoipupdate, fixes #591 

Checklist:
- [x] docs up-to-date

include/mua.sh

@@ -40,22 +40,26 @@ test_imap_openssl()
 EOF
 }
 
-test_imap_curl()
+get_curl_cmd()
 {
-	local _test_uri
-	_test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
-
 	if [ -x /usr/local/bin/curl ]; then
-		curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+		echo "curl"
 	elif [ -x "$STAGE_MNT/usr/local/bin/curl" ]; then
-		_test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@localhost/"
-		stage_exec curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+		echo "stage_exec curl"
 	else
-		pkg install -y curl
-		curl -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+		pkg install -qy curl
+		echo "curl"
 	fi
 }
 
+test_imap_curl()
+{
+	local _test_uri
+	_test_uri="imaps://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+	_curl_cmd="$(get_curl_cmd)"
+	$_curl_cmd -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
+}
+
 test_imap()
 {
 	echo "testing IMAP AUTH as $MUA_TEST_USER"
@@ -90,9 +94,11 @@ test_pop3_empty()
 
 test_pop3()
 {
+	local _test_uri
+	_test_uri="pop3s://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+	_curl_cmd="$(get_curl_cmd)"
 	# shellcheck disable=2001
-	curl -k -v --login-options 'AUTH=PLAIN' \
-		"pop3s://$(uriencode $MUA_TEST_USER):$(uriencode $MUA_TEST_PASS)@${MUA_TEST_HOST}/"
+	$_curl_cmd -k -v --login-options 'AUTH=PLAIN' "$_test_uri"
 }
 
 uriencode() {

include/nginx.sh

@@ -37,8 +37,10 @@ install_nginx()
 
 install_nginx_newsyslog()
 {
+	stage_enable_newsyslog
+
 	tell_status "enabling nginx log file rotation"
-	tee "$STAGE_MNT/etc/newsyslog.conf.d/nginx" <<EO_NG_NSL
+	tee "$STAGE_MNT/etc/newsyslog.conf.d/nginx.conf" <<EO_NG_NSL
 # rotate nightly (default)
 /var/log/nginx/*.log		root:wheel	644	 7     *   @T00   BCGX  /var/run/nginx.pid 30
 
@@ -69,10 +71,20 @@ configure_nginx_server_d()
 	fi
 
 	# most calls get enclosing server block
-	local _prefix='server {
-		listen       80 proxy_protocol;
-		listen  [::]:80 proxy_protocol;
+	local _prefix
+	if [ "$TOASTER_WEBMAIL_PROXY" = "haproxy" ]; then
+		_prefix='server {
+			listen       80;
+			listen  [::]:80;
+'
+	else
+		# nginx can't send proxy protocol AND route URIs at the same time
+		_prefix='server {
+			listen       80;
+			listen  [::]:80;
 '
+	fi
+
 	local _suffix='location ~ /\.ht {
 			deny  all;
 		}
@@ -136,8 +148,10 @@ http {
 	keepalive_timeout  65;
 
 	set_real_ip_from $(get_jail_ip haproxy);
+	set_real_ip_from $(get_jail_ip webmail);
 	set_real_ip_from $(get_jail_ip6 haproxy);
-	real_ip_header   proxy_protocol;
+	set_real_ip_from $(get_jail_ip6 webmail);
+	real_ip_header   X-Forwarded-For;
 	real_ip_recursive on;
 	client_max_body_size 25m;
 

include/php.sh

@@ -29,8 +29,10 @@ install_php()
 }
 
 install_php_newsyslog() {
+	stage_enable_newsyslog
+
 	tell_status "enabling PHP-FPM log file rotation"
-	tee "$STAGE_MNT/etc/newsyslog.conf.d/php-fpm" <<EO_FPM_NSL
+	tee "$STAGE_MNT/etc/newsyslog.conf.d/php-fpm.conf" <<EO_FPM_NSL
 # rotate the file after it reaches 1M
 /var/log/php-fpm.log 600 7	1024	*	BCX	/var/run/php-fpm.pid 30
 EO_FPM_NSL

include/shell.sh

@@ -51,7 +51,7 @@ export HISTIGNORE="&:[bf]g:exit"
 
 shopt -s histappend
 shopt -s cdspell
-set -o vi
+#set -o vi
 
 if [[ $- == *i* ]]
 then

mail-toaster.sh

@@ -88,7 +88,10 @@ export TOASTER_PKG_BRANCH="latest"
 export TOASTER_USE_TMPFS="0"
 export TOASTER_VPOPMAIL_CLEAR="1"
 export TOASTER_VPOPMAIL_EXT="0"
+export TOASTER_WEBMAIL_PROXY="haproxy"
 export CLAMAV_FANGFRISCH="0"
+export GEOIP_UPDATER="geoipupdate"
+export MAXMIND_ACCOUNT_ID=""
 export MAXMIND_LICENSE_KEY=""
 export ROUNDCUBE_SQL="0"
 export ROUNDCUBE_DEFAULT_HOST=""
@@ -188,6 +191,7 @@ export TOASTER_USE_TMPFS=${TOASTER_USE_TMPFS:="0"}
 export TOASTER_VPOPMAIL_CLEAR=${TOASTER_VPOPMAIL_CLEAR:="1"}
 export TOASTER_VPOPMAIL_EXT=${TOASTER_VPOPMAIL_EXT:="0"}
 export TOASTER_VQADMIN=${TOASTER_VQADMIN:="0"}
+export TOASTER_WEBMAIL_PROXY=${TOASTER_WEBMAIL_PROXY:="haproxy"}
 export CLAMAV_FANGFRISCH=${CLAMAV_FANGFRISCH:="0"}
 export CLAMAV_UNOFFICIAL=${CLAMAV_UNOFFICIAL:="0"}
 export ROUNDCUBE_SQL=${ROUNDCUBE_SQL:="$TOASTER_MYSQL"}
@@ -1012,6 +1016,38 @@ stage_fbsd_package()
 	echo "done"
 }
 
+stage_setup_tls()
+{
+	# static TLS certificates (installed at deploy)
+	if [ ! -f "$STAGE_MNT/etc/ssl/certs/${TOASTER_MAIL_DOMAIN}.pem" ]; then
+		tell_status "installing TLS certificate"
+		cp /etc/ssl/certs/server.crt "$STAGE_MNT/etc/ssl/certs/${TOASTER_MAIL_DOMAIN}.pem"
+		cp /etc/ssl/private/server.key "$STAGE_MNT/etc/ssl/private/${TOASTER_MAIL_DOMAIN}.pem"
+	fi
+
+	# dynamic TLS certs, kept up-to-date by acme.sh or certbot
+	if [ ! -f "$STAGE_MNT/data/etc/tls/certs" ]; then
+		# shellcheck disable=SC2174
+		mkdir -m 0644 -p "$STAGE_MNT/data/etc/tls/certs"
+		cp /etc/ssl/certs/server.crt "$STAGE_MNT/data/etc/tls/certs/${TOASTER_MAIL_DOMAIN}.pem"
+	fi
+
+	if [ ! -f "$STAGE_MNT/data/etc/tls/private" ]; then
+		# shellcheck disable=SC2174
+		mkdir -m 0640 -p "$STAGE_MNT/data/etc/tls/private"
+		cp /etc/ssl/private/server.key "$STAGE_MNT/data/etc/tls/private/${TOASTER_MAIL_DOMAIN}.pem"
+	fi
+}
+
+stage_enable_newsyslog()
+{
+	tell_status "enabling newsyslog"
+	sysrc -f "$STAGE_MNT/etc/rc.conf" newsyslog_enable=YES
+	sed -i.bak \
+		-e '/^#0.*newsyslog/ s/^#0/0/' \
+		"$STAGE_MNT/etc/crontab"
+}
+
 unmount_data()
 {
 	# $1 is ZFS fs (eg: /data/mysql)
@@ -1348,7 +1384,8 @@ assure_jail()
 	fi
 }
 
-preserve_file() {
+preserve_file()
+{
 	local _jail_name=$1
 	local _file_path=$2
 

provision/base.sh

@@ -125,7 +125,7 @@ do
 done
 
 # packages to be updated automatically
-auto_upgrade="curl expat libxml2 pkg sudo vim-tiny"
+auto_upgrade="curl expat libxml2 pkg sudo unbound vim-tiny"
 
 # add packages with:
 #   sysrc -f /usr/local/etc/periodic/daily/auto_security_upgrades auto_upgrade+=" $NEW"
@@ -137,17 +137,15 @@ done
 EO_PKG_SECURITY
 }
 
-configure_ssl_dirs()
+configure_tls_dirs()
 {
 	if [ ! -d "$BASE_MNT/etc/ssl/certs" ]; then
-		mkdir "$BASE_MNT/etc/ssl/certs"
+		mkdir -m 0644 "$BASE_MNT/etc/ssl/certs"
 	fi
 
 	if [ ! -d "$BASE_MNT/etc/ssl/private" ]; then
-		mkdir "$BASE_MNT/etc/ssl/private"
+		mkdir -m 0640 "$BASE_MNT/etc/ssl/private"
 	fi
-
-	chmod o-r "$BASE_MNT/etc/ssl/private"
 }
 
 configure_tls_dhparams()
@@ -215,7 +213,7 @@ configure_base()
 		update_motd=NO
 
 	configure_pkg_latest "$BASE_MNT"
-	configure_ssl_dirs
+	configure_tls_dirs
 	configure_tls_dhparams
 	disable_cron_jobs
 	enable_security_periodic

provision/dovecot.sh

@@ -112,13 +112,13 @@ protocol lmtp {
 }
 
 # default TLS certificate (no SNI)
-ssl_cert = </data/etc/ssl/certs/dovecot.pem
-ssl_key = </data/etc/ssl/private/dovecot.pem
+ssl_cert = </data/etc/tls/certs/dovecot.pem
+ssl_key = </data/etc/tls/private/dovecot.pem
 
 # example TLS SNI (see https://wiki.dovecot.org/SSL/DovecotConfiguration)
 #local_name mail.example.com {
-#  ssl_cert = </data/etc/ssl/certs/mail.example.com.pem
-#  ssl_key = </data/etc/ssl/private/mail.example.com.pem
+#  ssl_cert = </data/etc/tls/certs/mail.example.com.pem
+#  ssl_key = </data/etc/tls/private/mail.example.com.pem
 #}
 
 # dovecot 2.3+ supports a ssl_dh file
@@ -322,7 +322,7 @@ configure_tls_certs()
 			"$_localconf"
 	fi
 
-	local _ssldir="$ZFS_DATA_MNT/dovecot/etc/ssl"
+	local _ssldir="$ZFS_DATA_MNT/dovecot/etc/tls"
 	if [ ! -d "$_ssldir/certs" ]; then
 		# shellcheck disable=SC2174
 		mkdir -m 644 -p "$_ssldir/certs"
@@ -351,8 +351,8 @@ configure_postfix_with_sasl()
 	stage_exec postconf -e 'smtpd_sasl_path = private/auth'
 	stage_exec postconf -e 'smtpd_sasl_auth_enable = yes'
 	stage_exec postconf -e 'smtpd_recipient_restrictions = permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination'
-	stage_exec postconf -e "smtpd_tls_cert_file = /data/etc/ssl/certs/$TOASTER_HOSTNAME.pem"
-	stage_exec postconf -e "smtpd_tls_key_file = /data/etc/ssl/private/$TOASTER_HOSTNAME.pem"
+	stage_exec postconf -e "smtpd_tls_cert_file = /data/etc/tls/certs/$TOASTER_HOSTNAME.pem"
+	stage_exec postconf -e "smtpd_tls_key_file = /data/etc/tls/private/$TOASTER_HOSTNAME.pem"
 	stage_exec postconf -e 'smtp_tls_security_level = may'
 
 	for _s in 512 1024 2048; do