customized fixes from PR#187

haiwen#187

scripts/bootstrap.py

@@ -17,8 +17,10 @@
     call, get_conf, get_install_dir, loginfo,
     get_script, render_template, get_seafile_version, eprint,
     cert_has_valid_days, get_version_stamp_file, update_version_stamp,
-    wait_for_mysql, wait_for_nginx, read_version_stamp
+    wait_for_mysql, read_version_stamp, set_key, listen_on_https
 )
+import utils.settings
+import utils.nginx
 
 seafile_version = get_seafile_version()
 installdir = get_install_dir()
@@ -29,7 +31,7 @@
 
 def init_letsencrypt():
     loginfo('Preparing for letsencrypt ...')
-    wait_for_nginx()
+    utils.nginx.wait_for_nginx()
 
     if not exists(ssl_dir):
         os.mkdir(ssl_dir)
@@ -56,17 +58,8 @@ def init_letsencrypt():
             return
 
     loginfo('Starting letsencrypt verification')
-    # Create a temporary nginx conf to start a server, which would accessed by letsencrypt
-    context = {
-        'https': False,
-        'domain': domain,
-    }
-    if not os.path.isfile('/shared/nginx/conf/seafile.nginx.conf'):
-        render_template('/templates/seafile.nginx.conf.template',
-                        '/etc/nginx/sites-enabled/seafile.nginx.conf', context)
-
-    call('nginx -s reload')
-    time.sleep(2)
+    # Create a temporary nginx conf to start a server, which would be accessed by letsencrypt
+    utils.nginx.change_nginx_config(False, True)
 
     call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain))
     # if call('/scripts/ssl.sh {0} {1}'.format(ssl_dir, domain), check_call=False) != 0:
@@ -78,27 +71,6 @@ def init_letsencrypt():
     # Create a crontab to auto renew the cert for letsencrypt.
 
 
-def generate_local_nginx_conf():
-    # Now create the final nginx configuratin
-    domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com')
-    context = {
-        'https': is_https(),
-        'domain': domain,
-    }
-
-    if not os.path.isfile('/shared/nginx/conf/seafile.nginx.conf'):
-        render_template(
-            '/templates/seafile.nginx.conf.template',
-            '/etc/nginx/sites-enabled/seafile.nginx.conf',
-            context
-        )
-        nginx_etc_file = '/etc/nginx/sites-enabled/seafile.nginx.conf'
-        nginx_shared_file = '/shared/nginx/conf/seafile.nginx.conf'
-        call('mv {0} {1} && ln -sf {1} {0}'.format(nginx_etc_file, nginx_shared_file))
-
-def is_https():
-    return get_conf('SEAFILE_SERVER_LETSENCRYPT', 'false').lower() == 'true'
-
 def parse_args():
     ap = argparse.ArgumentParser()
     ap.add_argument('--parse-ports', action='store_true')
@@ -119,16 +91,8 @@ def init_seafile_server():
         return
 
     loginfo('Now running setup-seafile-mysql.py in auto mode.')
-    env = {
-        'SERVER_NAME': 'seafile',
-        'SERVER_IP': get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com'),
-        'MYSQL_USER': 'seafile',
-        'MYSQL_USER_PASSWD': str(uuid.uuid4()),
-        'MYSQL_USER_HOST': '%.%.%.%',
-	'MYSQL_HOST': get_conf('DB_HOST','127.0.0.1'),
-        # Default MariaDB root user has empty password and can only connect from localhost.
-        'MYSQL_ROOT_PASSWD': get_conf('DB_ROOT_PASSWD', ''),
-    }
+
+    env = utils.settings.from_environment()
 
     # Change the script to allow mysql root password to be empty
     # call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}'''
@@ -141,28 +105,22 @@ def init_seafile_server():
     call('''sed -i -e '/def validate_mysql_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}'''
         .format(get_script('setup-seafile-mysql.py')))
 
+
+    # Change SQL for seahub db to not fail if tables or records are there. #https://github.com/haiwen/seafile-server/issues/188
+    call('''sed -i -Ee 's@(CREATE TABLE\s+)`@\\1 IF NOT EXISTS `@gi' {}'''
+        .format(get_script('seahub/sql/mysql.sql')))
+    call('''sed -i -Ee 's@INSERT INTO\s+`@INSERT IGNORE `@gi' {}'''
+        .format(get_script('seahub/sql/mysql.sql')))
+
     setup_script = get_script('setup-seafile-mysql.sh')
+    #logdbg("  env is: " + str(env))
     call('{} auto -n seafile'.format(setup_script), env=env)
 
-    domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com')
-    proto = 'https' if is_https() else 'http'
-    with open(join(topdir, 'conf', 'seahub_settings.py'), 'a+') as fp:
-        fp.write('\n')
-        fp.write("""CACHES = {
-    'default': {
-        'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
-        'LOCATION': 'memcached:11211',
-    },
-    'locmem': {
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-    },
-}
-COMPRESS_CACHE_BACKEND = 'locmem'""")
-        fp.write('\n')
-        fp.write("TIME_ZONE = '{time_zone}'".format(time_zone=os.getenv('TIME_ZONE',default='Etc/UTC')))
-        fp.write('\n')
-        fp.write('FILE_SERVER_ROOT = "{proto}://{domain}/seafhttp"'.format(proto=proto, domain=domain))
-        fp.write('\n')
+
+    settings = utils.settings.read_them() # previous call might have written something already
+    utils.settings.update_from_env(settings, env)
+    utils.settings.write_them(settings)
+
 
     # By default ccnet-server binds to the unix socket file
     # "/opt/seafile/ccnet/ccnet.sock", but /opt/seafile/ccnet/ is a mounted

scripts/seafdav.patch

@@ -0,0 +1,31 @@
+*** seafile-server-7.0.4.orig/seahub/thirdpart/wsgidav/addons/seafile/seahub_db.py      2019-05-28 04:13:37.000000000 +0000
+--- seafile-server-7.0.4/seahub/thirdpart/wsgidav/addons/seafile/seahub_db.py   2019-10-30 18:19:08.451625194 +0000
+*************** from sqlalchemy.orm import sessionmaker
+*** 7,12 ****
+--- 7,16 ----
+  from sqlalchemy.pool import Pool
+  from sqlalchemy.ext.automap import automap_base
+
++ from seaf_utils import SEAFILE_CENTRAL_CONF_DIR, SEAFILE_CONF_DIR
++ import os
++ import sys
++ 
+  Base = automap_base()
+
+  import wsgidav.util as util
+*************** def init_db_session_class():
+*** 24,29 ****
+--- 28,39 ----
+          return None
+
+  def create_seahub_db_engine():
++     path = SEAFILE_CENTRAL_CONF_DIR if SEAFILE_CENTRAL_CONF_DIR else SEAFILE_CONF_DIR;
++     if os.path.exists(path):
++         sys.path.insert(0, path)
++ 
++ 
++ 
+      import seahub_settings
+      db_infos = seahub_settings.DATABASES['default']
+      #import local_settings
+

scripts/start.py

@@ -12,13 +12,18 @@
 import shutil
 import sys
 import time
+import re
+import ConfigParser
 
 from utils import (
     call, get_conf, get_install_dir, get_script, get_command_output,
-    render_template, wait_for_mysql, setup_logging
+    render_template, wait_for_mysql, setup_logging, listen_on_https
 )
+import utils.settings
+import utils.nginx
+
 from upgrade import check_upgrade
-from bootstrap import init_seafile_server, is_https, init_letsencrypt, generate_local_nginx_conf
+from bootstrap import init_seafile_server, init_letsencrypt
 
 
 shared_seafiledir = '/shared/seafile'
@@ -41,21 +46,59 @@ def watch_controller():
     print 'seafile controller exited unexpectedly.'
     sys.exit(1)
 
+
+def apply_code_fixes():
+    pass
+    # # fix seafdav not starting
+    # call('''cd {0}; patch --forward -p 1 < /scripts/seafdav.patch || true'''.format(get_install_dir()))
+
+
+# environment might have changed (db names, memcached hostname, etc
+def update_settings():
+    settings = utils.settings.read_them()
+    env = utils.settings.from_environment()
+
+    utils.settings.update_from_env(settings, env)
+    utils.settings.write_them(settings)
+
+
+def update_seafdav_config():
+    f = os.path.join(topdir, 'conf', 'seafdav.conf')
+    if os.path.exists(f):
+        cp = ConfigParser.ConfigParser()
+        cp.read(f)
+        section_name = 'WEBDAV'
+        cp.set(section_name, 'share_name', '/seafdav')
+        cp.set(section_name, 'fastcgi', 'false')
+        cp.set(section_name, 'port', '8080')
+        cp.set(section_name, 'enabled', "true" if (get_conf('ENABLE_WEBDAV', '0') != '0') else "false")
+        with open(f, "w") as fp:
+            cp.write(fp)
+
+
+
+
+
 def main():
     if not exists(shared_seafiledir):
         os.mkdir(shared_seafiledir)
     if not exists(generated_dir):
         os.makedirs(generated_dir)
 
-    if is_https():
+    if listen_on_https():
         init_letsencrypt()
-    generate_local_nginx_conf()
-    call('nginx -s reload')
+    utils.nginx.wait_for_nginx()
+    utils.nginx.change_nginx_config()
 
     wait_for_mysql()
     init_seafile_server()
 
     check_upgrade()
+
+    apply_code_fixes()
+    update_settings()
+    update_seafdav_config()
+
     os.chdir(installdir)
 
     admin_pw = {

scripts/utils/__init__.py

@@ -19,7 +19,17 @@
 
 logger = logging.getLogger('.utils')
 
-DEBUG_ENABLED = os.environ.get('SEAFILE_DOCKER_VERBOSE', '').lower() in ('true', '1', 'yes')
+
+def get_conf(key, default=None):
+    key = key.upper()
+    return os.environ.get(key, default)
+
+def get_conf_bool(key, default="false"):
+    v = get_conf(key, default)
+    return v.lower() in ('true', '1', 'yes')
+
+
+DEBUG_ENABLED = get_conf_bool('SEAFILE_DOCKER_VERBOSE','')
 
 def eprint(*a, **kw):
     kw['file'] = sys.stderr
@@ -51,7 +61,7 @@ def _find_flag(args, *opts, **kw):
 
 def call(*a, **kw):
     dry_run = kw.pop('dry_run', False)
-    quiet = kw.pop('quiet', DEBUG_ENABLED)
+    quiet = kw.pop('quiet', not DEBUG_ENABLED)
     cwd = kw.get('cwd', os.getcwd())
     check_call = kw.pop('check_call', True)
     reduct_args = kw.pop('reduct_args', [])
@@ -218,10 +228,7 @@ def get_script(script):
 
 _config = None
 
-def get_conf(key, default=None):
-    key = key.upper()
-    return os.environ.get(key, default)
-
+
 def _add_default_context(context):
     default_context = {
         'current_timestr': datetime.datetime.now().strftime('%m/%d/%Y %H:%M:%S'),
@@ -267,8 +274,12 @@ def update_version_stamp(version, fn=get_version_stamp_file()):
 
 def wait_for_mysql():
     db_host = get_conf('DB_HOST', '127.0.0.1')
-    db_user = 'root'
-    db_passwd = get_conf('DB_ROOT_PASSWD', '')
+    if get_conf('USE_EXISTING_DB', '0') == '1':
+       db_user = get_conf('DB_USER', '')
+       db_passwd = get_conf('DB_PASSWD', '')
+    else:
+       db_user = 'root'
+       db_passwd = get_conf('DB_ROOT_PASSWD', '')
 
     while True:
         try:
@@ -280,18 +291,39 @@ def wait_for_mysql():
 	logdbg('mysql server is ready')
 	return
 
-def wait_for_nginx():
-    while True:
-        logdbg('waiting for nginx server to be ready')
-        output = get_command_output('netstat -nltp')
-        if ':80 ' in output:
-            logdbg(output)
-            logdbg('nginx is ready')
-            return
-        time.sleep(2)
 
 def replace_file_pattern(fn, pattern, replacement):
     with open(fn, 'r') as fp:
         content = fp.read()
     with open(fn, 'w') as fp:
         fp.write(content.replace(pattern, replacement))
+
+def ensure_dict(dictionary, keys):
+    d = dictionary
+    for k in keys:
+        if k not in d:
+            d[k] = {}
+        d = d[k]
+
+def set_key(dictionary, keys, v):
+    if not isinstance(keys, (list,tuple)):
+        keys = [keys]
+    d = dictionary
+    for k in keys[:-1]:
+        if k not in d:
+            d[k] = {}
+        d = d[k]
+    d[keys[-1]] = v
+
+
+def behind_ssl_termination():
+    return get_conf_bool('BEHIND_SSL_TERMINATION')
+
+def listen_on_https():
+    return not behind_ssl_termination() and get_conf_bool('SEAFILE_SERVER_LETSENCRYPT', 'false')
+
+
+def uses_https():
+    return listen_on_https() or behind_ssl_termination()
+
+

scripts/utils/nginx.py

@@ -0,0 +1,42 @@
+import time
+import os
+
+from utils import (get_command_output, logdbg, listen_on_https,
+                   behind_ssl_termination, get_conf, render_template, call)
+
+
+def wait_for_nginx():
+    while True:
+        logdbg('waiting for nginx server to be ready')
+        output = get_command_output('netstat -nltp')
+        if ':80 ' in output:
+            logdbg(output)
+            logdbg('nginx is ready')
+            return
+        time.sleep(2)
+
+
+def change_nginx_config(https=None, skip_writing_shared_conf=False):
+    if https is None:
+        https = listen_on_https()
+
+    domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com')
+
+    nginx_shared_file = '/shared/nginx/conf/seafile.nginx.conf'
+
+    if not os.path.isfile(nginx_shared_file):
+        nginx_etc_file = '/etc/nginx/sites-enabled/seafile.nginx.conf'
+        context = {
+            'https': https,
+            # 'behind_ssl_termination': behind_ssl_termination(),
+            'domain': domain,
+            # 'enable_webdav': get_conf('ENABLE_WEBDAV', '0') != '0'
+        }
+        render_template('/templates/seafile.nginx.conf.template',
+                        nginx_etc_file, context)
+        if not skip_writing_shared_conf:
+            call(
+                'mv {0} {1} && ln -sf {1} {0}'.format(nginx_etc_file, nginx_shared_file))
+
+    call('nginx -s reload')
+    time.sleep(2)