[ADD] runbot_preseed_database

.travis.yml

@@ -25,8 +25,8 @@ env:
   - VERSION="11.0" TESTS="0" LINT_CHECK="0"
   matrix:
   - LINT_CHECK="1"
-  - TESTS="1" ODOO_REPO="OCA/OCB"
-  - TESTS="1" ODOO_REPO="odoo/odoo"
+  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="runbot_buildout"
+  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="runbot_travis2docker"
 
 
 install:
@@ -35,6 +35,7 @@ install:
   - travis_install_nightly
 
 script:
+  # TODO: add some magic to skip buildout and/or travis2docker builds if none of them changed
   - travis_run_tests
 
 after_success:

runbot_preseed_database/README.rst

@@ -0,0 +1,63 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :target: https://www.gnu.org/licenses/agpl-3.0-standalone.html
+    :alt: License: AGPL-3
+
+===============================
+Preseed test database in runbot
+===============================
+
+This module allows you to have runbot use a database template that already has some modules installed. This allows for much faster builds in case you're only interested in a subset of the tests.
+
+Configuration
+=============
+
+To configure this module, you need to:
+
+#. go to your repository (all builds in the repo) or a branch (just builds for this branch)
+#. alternatively
+   - fill in a database name to be used as template. This database must exist in the cluster and be accessible for the user running runbot
+   - use the wizard to select a build to use to generate a template. This will also preselect all modules installed in the build's database minus the build's modules to test as the list of modules to preseed
+#. when you use builds to generate template, review the refresh cronjob to move it to a time where regeneration won't interfere with your development. During refreshing, repos and branches with preseeded database will fall back to the empty database, so you will have slow builds in this period, not failing ones
+
+Usage
+=====
+
+You will have to regenerate the database template from time if you don't use the build mechanism, and you should also take care to have one branch tested with a fresh database. This is meant to speedup builds for which most tests are irrelevant.
+
+Bug Tracker
+===========
+
+Bugs are tracked on `GitHub Issues
+<https://github.com/OCA/runbot-addons/issues>`_. In case of trouble, please
+check there if your issue has already been reported. If you spotted it first,
+help us smashing it by providing a detailed and welcomed feedback.
+
+Credits
+=======
+
+Images
+------
+
+* Odoo Community Association: `Icon <https://github.com/OCA/maintainer-tools/blob/master/template/module/static/description/icon.svg>`_.
+
+Contributors
+------------
+
+* Holger Brunn <[email protected]>
+
+Do not contact contributors directly about help with questions or problems concerning this addon, but use the `community mailing list <mailto:[email protected]>`_ or the `appropriate specialized mailinglist <https://odoo-community.org/groups>`_ for help, and the bug tracker linked in `Bug Tracker`_ above for technical issues.
+
+Maintainer
+----------
+
+.. image:: https://odoo-community.org/logo.png
+   :alt: Odoo Community Association
+   :target: https://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose
+mission is to support the collaborative development of Odoo features and
+promote its widespread use.
+
+To contribute to this module, please visit https://odoo-community.org.

runbot_preseed_database/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+from . import models
+from . import wizards

runbot_preseed_database/__manifest__.py

@@ -0,0 +1,20 @@
+# Copyright 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+{
+    "name": "Preseed test database in runbot",
+    "version": "11.0.1.0.0",
+    "author": "Therp BV,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Runbot",
+    "summary": "Allows to run tests with a partially initialized database",
+    "depends": [
+        'runbot',
+    ],
+    "data": [
+        "data/ir_cron.xml",
+        "security/ir.model.access.csv",
+        "wizards/runbot_preseed_database_refresh.xml",
+        "views/runbot_branch.xml",
+        "views/runbot_repo.xml",
+    ],
+}

runbot_preseed_database/data/ir_cron.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<odoo noupdate="1">
+    <record id="server_action_cron" model="ir.actions.server">
+        <field name="name">Regenerate preseed databases</field>
+        <field name="state">code</field>
+        <field name="model_id" ref="runbot_preseed_database.model_runbot_preseed_database_refresh" />
+        <field name="code">model._refresh_preseed_database()</field>
+    </record>
+    <record id="cron" model="ir.cron">
+        <field name="ir_actions_server_id" ref="server_action_cron" />
+        <field name="interval_type">months</field>
+        <field name="numbercall">-1</field>
+    </record>
+</odoo>

runbot_preseed_database/models/__init__.py

@@ -0,0 +1,6 @@
+# Copyright 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+from . import runbot_repo
+from . import runbot_build
+from . import runbot_branch
+from . import runbot_preseed_database_module

runbot_preseed_database/models/runbot_branch.py

@@ -0,0 +1,24 @@
+# © 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+from odoo import api, fields, models
+
+
+class RunbotBranch(models.Model):
+    _inherit = 'runbot.branch'
+
+    preseed_database = fields.Char(
+        help='Fill in the name of a database to use as template for the all '
+        'build', copy=False,
+    )
+    preseed_database_module_ids = fields.Many2many(
+        'runbot.preseed.database.module', string='Modules to install',
+        copy=False,
+    )
+    preseed_database_build_id = fields.Many2one(
+        'runbot.build', string='Preseed database build', ondelete='restrict',
+        help='Select a build to use to generate the preseed database',
+    )
+
+    @api.constrains('preseed_database')
+    def _check_preseed_database(self):
+        self.env['runbot.repo']._check_preseed_database.__func__(self)

runbot_preseed_database/models/runbot_build.py

@@ -0,0 +1,149 @@
+# © 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+import inspect
+import os
+import shutil
+from psycopg2.extensions import AsIs
+from odoo import api, fields, models
+from odoo.addons.runbot.common import local_pgadmin_cursor
+
+
+class RunbotBuild(models.Model):
+    _inherit = 'runbot.build'
+
+    preseed_database = fields.Char(
+        compute=lambda self: [
+            this.update({
+                'preseed_database':
+                this.branch_id.preseed_database or
+                this.branch_id.repo_id.preseed_database
+            })
+            for this in self
+        ],
+    )
+    preseed_database_build_id = fields.Many2one(
+        'runbot.build',
+        compute=lambda self: [
+            this.update({
+                'preseed_database_build_id':
+                this.branch_id.preseed_database_build_id or
+                this.branch_id.repo_id.preseed_database_build_id
+            })
+            for this in self
+        ],
+    )
+    preseed_repo_ids = fields.One2many(
+        'runbot.repo', 'preseed_database_build_id',
+    )
+    preseed_branch_ids = fields.One2many(
+        'runbot.branch', 'preseed_database_build_id',
+    )
+
+    def _job_00_init(self, build, lock_path, log_path):
+        """Don't do a checkout for preseed builds"""
+        if build.preseed_repo_ids or build.preseed_branch_ids:
+            build._log('init', 'Skipping checkout for database refresh')
+            build._github_status()
+            return -2
+        return super(RunbotBuild, self)._job_00_init(
+            build, lock_path, log_path
+        )
+
+    def _job_10_test_base(self, build, lock_path, log_path):
+        """No need for base tests for preseed builds"""
+        if build.preseed_repo_ids or build.preseed_branch_ids:
+            build._log('test_base', 'Skipping base test for database refresh')
+            return -2
+        return super(RunbotBuild, self)._job_10_test_base(
+            build, lock_path, log_path
+        )
+
+    def _job_30_run(self, build, lock_path, log_path):
+        """Don't run preseed builds, this locks the database"""
+        if build.preseed_repo_ids or build.preseed_branch_ids:
+            build._log('test_base', 'Skipping running preseed build')
+            return -2
+        return super(RunbotBuild, self)._job_30_run(
+            build, lock_path, log_path
+        )
+
+    def _local_pg_createdb(self, dbname):
+        """Use preseed database if applicable"""
+        if dbname.endswith('-base'):
+            # no need to do anything here
+            return super(RunbotBuild, self)._local_pg_createdb(dbname)
+        stack = inspect.stack()
+        # there should be a build in the callee, don't recurse through stack
+        build = stack[1][0].f_locals.get('build')
+        if (
+                isinstance(build, models.BaseModel) and
+                build.preseed_database and
+                build.preseed_database_build_id != build and
+                (
+                    not build.preseed_database_build_id or
+                    build.preseed_database_build_id.state == 'done'
+                )
+        ):
+            build._log(
+                'init',
+                'Using %s as database template' % build.preseed_database
+            )
+            with local_pgadmin_cursor() as cr:
+                cr.execute(
+                    'create database "%s" TEMPLATE "%s"',
+                    (AsIs(dbname), AsIs(build.preseed_database)),
+                )
+            if build.preseed_database_build_id:
+                # copy the build's filestore
+                template_filestore = build.preseed_database_build_id._path(
+                    os.path.join(
+                        'datadir', 'filestore', build.preseed_database
+                    )
+                )
+                build._log(
+                    'init', 'Copying filestore from %s' % template_filestore
+                )
+                filestore = build._path(os.path.join('datadir', 'filestore'))
+                os.makedirs(filestore, exist_ok=True)
+                if os.path.exists(template_filestore):
+                    shutil.copytree(template_filestore, os.path.join(
+                        filestore, dbname
+                    ))
+        else:
+            super(RunbotBuild, self)._local_pg_createdb(dbname)
+
+    @api.multi
+    def _local_cleanup(self):
+        """Never clean up a preseed build"""
+        preseed_builds = self.env['runbot.build'].search([
+            '|',
+            ('preseed_repo_ids', '!=', False),
+            ('preseed_branch_ids', '!=', False),
+        ])
+        build2state = {build: build.state for build in preseed_builds}
+        # super deletes builds in state done
+        preseed_builds.write({'state': 'running'})
+        super(RunbotBuild, self - preseed_builds)._local_cleanup()
+        for build, state in build2state.items():
+            build.write({'state': state})
+
+    @api.multi
+    def reset(self):
+        self._reset_for_preseed()
+        return super(RunbotBuild, self).reset()
+
+    @api.multi
+    def _reset_for_preseed(self):
+        for this in self:
+            repo_or_branch = this.preseed_branch_ids or this.preseed_repo_ids
+            if not repo_or_branch:
+                continue
+            this.write({
+                'modules': ','.join(
+                    repo_or_branch.mapped('preseed_database_module_ids.name')
+                ),
+            })
+            self.env['ir.logging'].search([
+                ('build_id', '=', this.id),
+            ]).unlink()
+            this._reset()

runbot_preseed_database/models/runbot_preseed_database_module.py

@@ -0,0 +1,10 @@
+# © 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+from odoo import fields, models
+
+
+class RunbotPreseedDatabaseModule(models.Model):
+    _name = 'runbot.preseed.database.module'
+    _description = 'Preseed database module'
+
+    name = fields.Char(required=True)

runbot_preseed_database/models/runbot_repo.py

@@ -0,0 +1,31 @@
+# © 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+import psycopg2
+from odoo import api, fields, models
+
+
+class RunbotRepo(models.Model):
+    _inherit = 'runbot.repo'
+
+    preseed_database = fields.Char(
+        help='Fill in the name of a database to use as template for the all '
+        'build', copy=False,
+    )
+    preseed_database_module_ids = fields.Many2many(
+        'runbot.preseed.database.module', string='Modules to install',
+        help='Fill in some modules with long running tests like stock or '
+        'accounting. Do not fill in modules here that should be tested.',
+        copy=False,
+    )
+    preseed_database_build_id = fields.Many2one(
+        'runbot.build', string='Preseed database build', ondelete='restrict',
+        help='Select a build to use to generate the preseed database',
+    )
+
+    @api.constrains('preseed_database')
+    def _check_preseed_database(self):
+        for this in self:
+            if not this.preseed_database:
+                continue
+            connection = psycopg2.connect('dbname=%s' % this.preseed_database)
+            connection.close()

runbot_preseed_database/security/ir.model.access.csv

@@ -0,0 +1,3 @@
+"id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink"
+access_runbot_preseed_database_module,access_runbot_preseed_database_module,model_runbot_preseed_database_module,runbot.group_runbot_admin,1,1,1,1
+access_runbot_preseed_database_module_user,access_runbot_preseed_database_module,model_runbot_preseed_database_module,runbot.group_user,1,0,0,0

runbot_preseed_database/tests/__init__.py

@@ -0,0 +1,3 @@
+# Copyright 2018 Therp BV <https://therp.nl>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
+from . import test_runbot_preseed_database