Merge pull request scrapinghub#732 from scrapinghub/more-options

More options: WebGL, HTML5 media, Media Source

docs/scripting-overview.rst

@@ -150,8 +150,13 @@ Browsing Options
   (it is requird for some websites because Webkit doesn't have localStorage
   available in Private Mode);
 * :ref:`splash-images-enabled` allows to turn OFF downloading of images;
-* :ref:`splash-plugins-enabled` allows to enable plugins (in default DOcker image
-  it enables Flash);
+* :ref:`splash-plugins-enabled` allows to enable plugins (in the default
+  Docker image it enables Flash);
 * :ref:`splash-resource-timeout` allows to drop slow or hanging requests
   to related resources after a timeout
-
+* :ref:`splash-indexeddb-enabled` allows to turn IndexedDB ON
+* :ref:`splash-webgl-enabled` allows to turn WebGL OFF
+* :ref:`splash-html5-media-enabled` allows to turn off HTML5 media
+  (e.g. playback of ``<video>`` tags).
+* :ref:`splash-media-source-enabled` allows to turn off Media Source Extension
+  API support

docs/scripting-ref.rst

@@ -218,6 +218,47 @@ to enable it.
 
 .. _IndexedDB: https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API
 
+.. _splash-webgl-enabled:
+
+splash.webgl_enabled
+--------------------
+
+Enable or disable WebGL_.
+
+**Signature:** ``splash.webgl_enabled = true/false``
+
+WebGL is enabled by default. Use ``splash.webgl_enabled = false``
+to disable it.
+
+.. _WebGL: https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API
+
+.. _splash-html5-media-enabled:
+
+splash.html5_media_enabled
+--------------------------
+
+Enable or disable HTML5 media, including HTML5 video and audio
+(e.g. ``<video>`` elements playback).
+
+**Signature:** ``splash.html5_media_enabled = true/false``
+
+HTML5 media is enabled by default. Use ``splash.html5_media_enabled = false``
+to disable it.
+
+.. _splash-media-source-enabled:
+
+splash.media_source_enabled
+---------------------------
+
+Enable or disable `Media Source Extensions API`_.
+
+**Signature:** ``splash.media_source_enabled = true/false``
+
+Media Source is enabled by default. Use ``splash.media_source_enabled = false``
+to disable it.
+
+.. _Media Source Extensions API: https://developer.mozilla.org/en-US/docs/Web/API/Media_Source_Extensions_API
+
 Methods
 ~~~~~~~
 

splash/browser_tab.py

@@ -22,6 +22,7 @@
 from splash.qtutils import (
     OPERATION_QT_CONSTANTS,
     MediaSourceEnabled,
+    MediaEnabled,
     WrappedSignal,
     qt2py,
     qurl2ascii,
@@ -32,7 +33,7 @@
 from splash.render_options import validate_size_str
 from splash.qwebpage import SplashQWebPage, SplashQWebView
 from splash.exceptions import JsError, OneShotCallbackError, ScriptError
-from splash.utils import to_bytes, get_id, traverse_data
+from splash.utils import to_bytes, get_id
 from splash.jsutils import (
     get_sanitized_result_js,
     SANITIZE_FUNC_JS,
@@ -188,28 +189,34 @@ def set_private_mode_enabled(self, val):
     get_indexeddb_enabled = webpage_option_getter(QWebSettings.OfflineStorageDatabaseEnabled)
     set_indexeddb_enabled = webpage_option_setter(QWebSettings.OfflineStorageDatabaseEnabled)
 
+    get_media_source_enabled = webpage_option_getter(MediaSourceEnabled)
+    set_media_source_enabled = webpage_option_setter(MediaSourceEnabled)
+
+    get_html5_media_enabled = webpage_option_getter(MediaEnabled)
+    set_html5_media_enabled = webpage_option_setter(MediaEnabled)
+
+    get_webgl_enabled = webpage_option_getter(QWebSettings.WebGLEnabled)
+    set_webgl_enabled = webpage_option_setter(QWebSettings.WebGLEnabled)
+
     def _set_default_webpage_options(self, web_page):
-        """
-        Set QWebPage options.
-        TODO: allow to customize them.
-        """
+        """ Set QWebPage options. TODO: allow to customize defaults. """
         settings = web_page.settings()
-        settings.setAttribute(QWebSettings.JavascriptEnabled, True)
         settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls, True)
 
-        # enable Media Source by default, at least to make html5test.com work
-        settings.setAttribute(MediaSourceEnabled, True)
-
         scroll_bars = Qt.ScrollBarAsNeeded if self.visible else Qt.ScrollBarAlwaysOff
         web_page.mainFrame().setScrollBarPolicy(Qt.Vertical, scroll_bars)
         web_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, scroll_bars)
 
         if self.visible:
             settings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True)
 
+        self.set_js_enabled(True)
         self.set_plugins_enabled(defaults.PLUGINS_ENABLED)
         self.set_response_body_enabled(defaults.RESPONSE_BODY_ENABLED)
         self.set_indexeddb_enabled(defaults.INDEXEDDB_ENABLED)
+        self.set_webgl_enabled(defaults.WEBGL_ENABLED)
+        self.set_html5_media_enabled(defaults.HTML5_MEDIA_ENABLED)
+        self.set_media_source_enabled(defaults.MEDIA_SOURCE_ENABLED)
 
     def _setup_webpage_events(self):
         main_frame = self.web_page.mainFrame()

splash/defaults.py

@@ -90,3 +90,13 @@
 
 # IndexedDB
 INDEXEDDB_ENABLED = False
+
+# HTML5 media
+HTML5_MEDIA_ENABLED = True
+
+# WebGL
+WEBGL_ENABLED = True
+
+# Media Source Extension API
+# it is enabled by default at least to make html5test.com work
+MEDIA_SOURCE_ENABLED = True

splash/qtrender_lua.py

@@ -136,13 +136,13 @@ def _command_result_to_pyresult(res):
 def lua_property(name):
     """ Decorator for marking methods that make attributes available to Lua """
     def decorator(meth):
-        def setter(method):
+        def lua_setter(method):
             meth._setter_method = method.__name__
             return method
 
         meth._is_lua_property = True
         meth._name = name
-        meth.lua_setter = setter
+        meth.lua_setter = lua_setter
         return meth
 
     return decorator
@@ -1155,6 +1155,36 @@ def get_indexeddb_enabled(self):
     def set_indexeddb_enabled(self, value):
         self.tab.set_indexeddb_enabled(bool(value))
 
+    @lua_property('media_source_enabled')
+    @command()
+    def get_media_source_enabled(self):
+        return self.tab.get_media_source_enabled()
+
+    @get_media_source_enabled.lua_setter
+    @command()
+    def set_media_source_enabled(self, enabled):
+        self.tab.set_media_source_enabled(bool(enabled))
+
+    @lua_property('html5_media_enabled')
+    @command()
+    def get_html5_media_enabled(self):
+        return self.tab.get_html5_media_enabled()
+
+    @get_html5_media_enabled.lua_setter
+    @command()
+    def set_html5_media_enabled(self, enabled):
+        self.tab.set_html5_media_enabled(bool(enabled))
+
+    @lua_property('webgl_enabled')
+    @command()
+    def get_webgl_enabled(self):
+        return self.tab.get_webgl_enabled()
+
+    @get_webgl_enabled.lua_setter
+    @command()
+    def set_webgl_enabled(self, enabled):
+        self.tab.set_webgl_enabled(bool(enabled))
+
     @lua_property('resource_timeout')
     @command()
     def get_resource_timeout(self):

splash/qtutils.py

@@ -102,7 +102,7 @@
 # Constant from https://github.com/annulen/webkit which is not available
 # in PyQT:
 MediaSourceEnabled = QWebSettings.Accelerated2dCanvasEnabled + 1
-
+MediaEnabled = QWebSettings.Accelerated2dCanvasEnabled + 2
 
 # A global reference must be kept to QApplication, otherwise the process will
 # segfault

splash/tests/test_execute.py

@@ -3777,24 +3777,119 @@ def test_default_value(self):
 
 
 class WebGLTest(BaseLuaRenderTest):
+    # WebGL detection code is from
+    # https://developer.mozilla.org/en-US/docs/Learn/WebGL/By_example/Detect_WebGL
+    DETECT_WEBGL_JS = """
+    function () {
+        var canvas = document.createElement("canvas");
+        var gl = canvas.getContext("webgl")
+                    || canvas.getContext("experimental-webgl");
+        return ~~(gl && gl instanceof WebGLRenderingContext);
+    }    
+    """
+
     def test_webgl(self):
-        # WebGL detection code is from
-        # https://developer.mozilla.org/en-US/docs/Learn/WebGL/By_example/Detect_WebGL
         resp = self.request_lua("""
-        function main(splash)
-            webgl_supported = splash:jsfunc([[
-                function () {
-                    var canvas = document.createElement("canvas");
-                    var gl = canvas.getContext("webgl")
-                                || canvas.getContext("experimental-webgl");
-                    return (gl && gl instanceof WebGLRenderingContext);
-                }
-            ]])
-            return webgl_supported()
+        function main(splash, args)
+            local webgl_supported = splash:jsfunc(args.js)
+            return {
+                supported = webgl_supported(),
+                enabled = splash.webgl_enabled
+            }
         end
-        """)
+        """, {'js': self.DETECT_WEBGL_JS})
         self.assertStatusCode(resp, 200)
-        self.assertEqual(resp.text, "True")
+        self.assertEqual(resp.json(), {
+            'supported': defaults.WEBGL_ENABLED,
+            'enabled': defaults.WEBGL_ENABLED,
+        })
+
+    def test_webgl_enable_disable(self):
+        for enabled in [False, True]:
+            resp = self.request_lua("""
+            function main(splash, args)
+                splash.webgl_enabled = args.enabled
+                local webgl_supported = splash:jsfunc(args.js)
+                return webgl_supported()
+            end
+            """, {'js': self.DETECT_WEBGL_JS, 'enabled': enabled})
+            self.assertStatusCode(resp, 200)
+            self.assertEqual(resp.json(), enabled)
+
+
+class Html5MediaTest(BaseLuaRenderTest):
+    # from https://stackoverflow.com/questions/3570502/how-to-check-for-html5-video-support
+    HTML5_VIDEO_SUPPORTED_JS = """
+    function() {return !!document.createElement('video').canPlayType}
+    """
+
+    def test_defaults(self):
+        resp = self.request_lua("""
+        function main(splash, args)
+            local html5_video_supported = splash:jsfunc(args.js)
+            return {
+                supported = html5_video_supported(),
+                enabled = splash.html5_media_enabled
+            }
+        end
+        """, {'js': self.HTML5_VIDEO_SUPPORTED_JS})
+        self.assertStatusCode(resp, 200)
+        self.assertEqual(resp.json(), {
+            'supported': defaults.HTML5_MEDIA_ENABLED,
+            'enabled': defaults.HTML5_MEDIA_ENABLED,
+        })
+
+    def test_enable_disable(self):
+        for enabled in [False, True]:
+            resp = self.request_lua("""
+            function main(splash, args)
+                splash.html5_media_enabled = args.enabled
+                local html5_video_supported = splash:jsfunc(args.js)
+                return {enabled=html5_video_supported()}
+            end
+            """, {'js': self.HTML5_VIDEO_SUPPORTED_JS, 'enabled': enabled})
+            self.assertStatusCode(resp, 200)
+            self.assertEqual(resp.json(), {'enabled': enabled})
+
+
+class MediaSourceTest(BaseLuaRenderTest):
+    # detection code is adapted from
+    # https://github.com/nickdesaulniers/netfix/blob/gh-pages/demo/bufferAll.html
+    DETECT_MSE_JS = """
+    function() { 
+        var mimeCodec = 'video/mp4; codecs="avc1.42E01E, mp4a.40.2"'; 
+        return !!('MediaSource' in window && 
+                   window.MediaSource && 
+                   MediaSource.isTypeSupported(mimeCodec));
+    }    
+    """
+    def test_defaults(self):
+        resp = self.request_lua("""
+        function main(splash, args)
+            local mse_supported = splash:jsfunc(args.js)
+            return {
+                supported = mse_supported(),
+                enabled = splash.media_source_enabled
+            }
+        end
+        """, {'js': self.DETECT_MSE_JS})
+        self.assertStatusCode(resp, 200)
+        self.assertEqual(resp.json(), {
+            'supported': defaults.MEDIA_SOURCE_ENABLED,
+            'enabled': defaults.MEDIA_SOURCE_ENABLED,
+        })
+
+    def test_enable_disable(self):
+        for enabled in [False, True]:
+            resp = self.request_lua("""
+            function main(splash, args)
+                splash.media_source_enabled = args.enabled
+                local mse_supported = splash:jsfunc(args.js)
+                return {enabled=mse_supported()}
+            end
+            """, {'js': self.DETECT_MSE_JS, 'enabled': enabled})
+            self.assertStatusCode(resp, 200)
+            self.assertEqual(resp.json(), {'enabled': enabled})
 
 
 class MouseEventsTest(BaseLuaRenderTest):