Merge pull request #66 from RedTurtle/cleanup_41

Features that could be removed because already in the core

.coveragerc

@@ -1,6 +1,9 @@
+[run]
+relative_files = True
+
 [report]
 include =
-    src/redturtle/*
+    */src/redturtle/*
 omit =
     */test*
     */upgrades*

.gitignore

@@ -1,6 +1,7 @@
 .DS_Store
 pyvenv.cfg
 .coverage
+coverage.json
 .python-version
 *.egg-info
 *.log

CHANGES.rst

@@ -26,6 +26,12 @@ Changelog
 
 - Plone 6.0 support
 
+- Remove Patch to avoid auto-closed tags in portal transforms: https://github.com/plone/Products.PortalTransforms/pull/43
+  [mamico]
+
+- Remove Patch for folderish types migration https://github.com/plone/plone.volto/pull/86
+  [mamico]
+
 - Handle custom search ranking with AdvancedQuery.
   [mamico]
 

README.rst

@@ -27,8 +27,8 @@ This is an helper package that setup a RedTurtle's Plone site ready to work with
     :target: https://pypi.org/project/redturtle.volto/
     :alt: License
 
-.. |coverage| image:: https://coveralls.io/repos/github/redturtle/redturtle.volto/badge.svg?branch=master
-    :target: https://coveralls.io/github/redturtle/redturtle.volto?branch=master
+.. |coverage| image:: https://coveralls.io/repos/github/RedTurtle/redturtle.volto/badge.svg?branch=master
+    :target: https://coveralls.io/github/RedTurtle/redturtle.volto?branch=master
     :alt: Coverage
 
 Features

base.cfg

@@ -70,10 +70,11 @@ eggs = coverage
 recipe = collective.recipe.template
 input = inline:
     #!/bin/bash
+    set -e
     export TZ=UTC
     ${buildout:directory}/bin/coverage run bin/test $*
     ${buildout:directory}/bin/coverage html
-    ${buildout:directory}/bin/coverage report -m --fail-under=40
+    ${buildout:directory}/bin/coverage report -m --fail-under=60
     # ${buildout:directory}/bin/coverage report -m --fail-under=90
     # Fail (exit status 1) if coverage returns exit status 2 (this happens
     # when test coverage is below 100%.

setup.py

@@ -58,8 +58,9 @@
         "collective.monkeypatcher",
         "collective.purgebyid",
         "kitconcept.seo>=2.0.0",
-        "plone.volto>3.1.0",
+        "plone.volto>=4.0.0",
         "plone.restapi>=8.16.1",
+        "Products.PortalTransforms>=3.2.0",
     ],
     extras_require={
         "advancedquery": [

src/redturtle/volto/__init__.py

@@ -1,57 +1,14 @@
 # -*- coding: utf-8 -*-
 """Init and utils."""
-from zope.i18nmessageid import MessageFactory
 from plone.app.content.browser.vocabulary import PERMISSIONS
-from plone.volto import upgrades
-from plone.volto.upgrades import MIGRATION
-from plone import api
-from OFS.interfaces import IOrderedContainer
+from zope.i18nmessageid import MessageFactory
 
 import logging
 
+
 logger = logging.getLogger(__name__)
 
 
 _ = MessageFactory("redturtle.volto")
 
 PERMISSIONS["plone.app.vocabularies.Keywords"] = "View"
-
-
-def rt_migrate_content_classes(context):
-    """
-    Migrate content created with collective.folderishtypes to plone.volto.
-
-    REMOVE WHEN THIS PR IS MERGED: https://github.com/plone/plone.volto/pull/86
-
-    """
-    interface = "collective.folderishtypes.interfaces.IFolderishType"
-    idxs = ["object_provides", "getObjPositionInParent"]
-    brains = api.content.find(
-        object_provides=interface, sort_on="getObjPositionInParent"
-    )
-    total_brains = len(brains)
-    logger.info(f"Migration: {total_brains} contents to be migrated.")
-    for idx, brain in enumerate(brains):
-        content = brain.getObject()
-        content_id = content.getId()
-        content.__class__ = MIGRATION[content.portal_type]
-        parent = content.aq_parent
-        ordered = IOrderedContainer(parent, None)
-        if ordered is not None:
-            order = ordered.getObjectPosition(content.getId())
-            if order == 1:
-                # can be the default one and we will lose the ordering
-                order = ordered.keys().index(content.getId())
-        parent._delOb(content_id)
-        parent._setOb(content_id, content)
-        content = parent[content_id]
-        ordered.moveObjectToPosition(content.getId(), order)
-        content.reindexObject(idxs=idxs)
-
-        if idx and idx % 100 == 0:
-            logger.info(f"Migration: {idx + 1} / {total_brains}")
-
-    logger.info("Migration from collective.folderishtypes to plone.volto complete")
-
-
-upgrades.migrate_content_classes = rt_migrate_content_classes

src/redturtle/volto/adapters/stringinterp.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from plone.stringinterp.adapters import BaseSubstitution
 from Products.CMFCore.interfaces import IContentish
-from zope.component import adapter
 from redturtle.volto import _
+from zope.component import adapter
 
 try:
     from plone.stringinterp import _ as stringinterp_mf

src/redturtle/volto/browser/find_blocks.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
+from Acquisition import aq_base
 from copy import deepcopy
-from Products.Five import BrowserView
 from plone import api
-from Acquisition import aq_base
 from plone.dexterity.utils import iterSchemata
+from Products.Five import BrowserView
 from zope.schema import getFieldsInOrder
 
-import logging
 import json
+import logging
+
 
 logger = logging.getLogger(__name__)
 

src/redturtle/volto/browser/fix_links.py

@@ -1,15 +1,16 @@
 # -*- coding: utf-8 -*-
-from Products.Five import BrowserView
-from plone import api
 from Acquisition import aq_base
+from plone import api
 from plone.dexterity.utils import iterSchemata
-from zope.schema import getFieldsInOrder
 from plone.restapi.interfaces import IFieldDeserializer
+from Products.Five import BrowserView
 from zope.component import queryMultiAdapter
+from zope.schema import getFieldsInOrder
 
+import json
 import logging
 import re
-import json
+
 
 logger = logging.getLogger(__name__)
 

src/redturtle/volto/browser/sitemap.py

@@ -5,6 +5,7 @@
 from Products.CMFCore.utils import getToolByName
 from Products.CMFPlone.interfaces import IPloneSiteRoot
 from zope.component import getUtility
+
 import datetime
 
 

src/redturtle/volto/interfaces.py

@@ -2,8 +2,8 @@
 from plone.restapi.controlpanels.interfaces import IControlpanel
 from redturtle.volto import _
 from zope.interface import Interface
-from zope.schema import Bool
 from zope.publisher.interfaces.browser import IDefaultBrowserLayer
+from zope.schema import Bool
 
 
 class IRedturtleVoltoLayer(IDefaultBrowserLayer):

src/redturtle/volto/monkey.py

@@ -1,88 +1,15 @@
 # -*- coding: utf-8 -*-
 from Acquisition import aq_base
-from lxml import etree
-from lxml import html
-from lxml.html.clean import Cleaner
+from plone.app.caching import purge
 from plone.app.event.base import dt_start_of_day
 from plone.app.event.recurrence import Occurrence
-from plone.app.caching import purge
 from plone.event.interfaces import IEventAccessor
 from plone.event.recurrence import recurrence_sequence_ical
 from plone.event.utils import pydt
-from plone.registry.interfaces import IRegistry
 from Products.CMFPlone.interfaces import IConstrainTypes
-from Products.CMFPlone.interfaces import IFilterSchema
-from Products.CMFPlone.utils import safe_encode
-from Products.PortalTransforms.libtransforms.utils import bodyfinder
-from Products.PortalTransforms.transforms.safe_html import hasScript
-from zope.component import getUtility
 from zope.globalrequest import getRequest
 
 import datetime
-import six
-
-
-def scrub_html(self, orig):
-    # append html tag to create a dummy parent for the tree
-    html_parser = html.HTMLParser(encoding="utf-8")
-    orig = safe_encode(orig)
-    tag = b"<html"
-    if tag in orig.lower():
-        # full html
-        tree = html.fromstring(orig, parser=html_parser)
-        strip_outer = bodyfinder
-    else:
-        # partial html (i.e. coming from WYSIWYG editor)
-        tree = html.fragment_fromstring(orig, create_parent=True, parser=html_parser)
-
-        def strip_outer(s):
-            return s[5:-6]
-
-    for elem in tree.iter(etree.Element):
-        if elem is not None:
-            for attrib, value in elem.attrib.items():
-                if hasScript(value):
-                    del elem.attrib[attrib]
-
-    registry = getUtility(IRegistry)
-    self.settings = registry.forInterface(IFilterSchema, prefix="plone")
-
-    valid_tags = self.settings.valid_tags
-    nasty_tags = [t for t in self.settings.nasty_tags if t not in valid_tags]
-    if six.PY2:
-        safe_attrs = [attr.decode() for attr in html.defs.safe_attrs]
-    else:
-        safe_attrs = [i for i in html.defs.safe_attrs]
-    safe_attrs.extend(self.settings.custom_attributes)
-    remove_script = "script" in nasty_tags and 1 or 0
-    cleaner = Cleaner(
-        kill_tags=nasty_tags,
-        remove_tags=[],
-        allow_tags=valid_tags,
-        page_structure=False,
-        safe_attrs_only=True,
-        safe_attrs=safe_attrs,
-        embedded=False,
-        remove_unknown_tags=False,
-        meta=False,
-        javascript=remove_script,
-        scripts=remove_script,
-        forms=False,
-        style=False,
-    )
-    try:
-        cleaner(tree)
-    except AssertionError:
-        # some VERY invalid HTML
-        return ""
-    # remove all except body or outer div
-    # changes in
-    # https://github.com/plone/Products.PortalTransforms/pull/43
-    if six.PY2:
-        result = etree.tostring(tree, encoding="utf-8", method="html").strip()
-    else:
-        result = etree.tounicode(tree, method="html").strip()
-    return strip_outer(result)
 
 
 def occurrences(self, range_start=None, range_end=None):

src/redturtle/volto/monkey.zcml

@@ -6,13 +6,6 @@
 
   <include package="collective.monkeypatcher" />
 
-  <monkey:patch
-      original="scrub_html"
-      replacement=".monkey.scrub_html"
-      class="Products.PortalTransforms.transforms.safe_html.SafeHTML"
-      description="This works around issue https://github.com/plone/Products.PortalTransforms/pull/43"
-      />
-
   <monkey:patch
       original="occurrences"
       replacement=".monkey.occurrences"

src/redturtle/volto/restapi/deserializer/dxfields.py

@@ -7,7 +7,11 @@
 from plone.restapi.deserializer.blocks import path2uid
 from plone.restapi.deserializer.dxfields import (
     DatetimeFieldDeserializer as DefaultDatetimeFieldDeserializer,
+)
+from plone.restapi.deserializer.dxfields import (
     RichTextFieldDeserializer as BaseRichTextDeserializer,
+)
+from plone.restapi.deserializer.dxfields import (
     TextLineFieldDeserializer as BaseTextLineDeserializer,
 )
 from plone.restapi.interfaces import IFieldDeserializer

src/redturtle/volto/restapi/serializer/blocks.py

@@ -12,6 +12,7 @@
 from zope.globalrequest import getRequest
 from zope.interface import implementer
 
+
 EXCLUDE_KEYS = ["@type"]
 EXCLUDE_TYPES = ["title", "listing"]
 

src/redturtle/volto/restapi/serializer/dxfields.py

@@ -1,21 +1,22 @@
 # -*- coding: utf-8 -*-
 from plone.app.contenttypes.interfaces import ILink
 from plone.app.contenttypes.utils import replace_link_variables_by_paths
+from plone.app.dexterity.behaviors.metadata import IPublication
+from plone.dexterity.interfaces import IDexterityContent
 from plone.outputfilters.browser.resolveuid import uuidToURL
+from plone.restapi.interfaces import IFieldSerializer
 from plone.restapi.serializer.converters import json_compatible
 from plone.restapi.serializer.dxfields import DefaultFieldSerializer
 from redturtle.volto.interfaces import IRedturtleVoltoLayer
 from zope.component import adapter
 from zope.component import getMultiAdapter
-from zope.schema.interfaces import ITextLine
-from plone.dexterity.interfaces import IDexterityContent
-from plone.restapi.interfaces import IFieldSerializer
 from zope.interface import implementer
 from zope.schema.interfaces import IDatetime
-from plone.app.dexterity.behaviors.metadata import IPublication
+from zope.schema.interfaces import ITextLine
 
 import re
 
+
 RESOLVEUID_RE = re.compile(".*?/resolve[Uu]id/([^/]*)/?(.*)$")
 
 

src/redturtle/volto/restapi/serializer/summary.py

@@ -16,6 +16,7 @@
 
 import re
 
+
 RESOLVEUID_RE = re.compile(".*?/resolve[Uu]id/([^/]*)/?(.*)$")
 
 EMPTY_STRINGS = ["None"]

src/redturtle/volto/restapi/services/controlpanel.py

@@ -1,12 +1,10 @@
 # -*- coding: utf-8 -*-
-from redturtle.volto.interfaces import (
-    IRedTurtleVoltoSettings,
-    IRedTurtleVoltoSettingsControlpanel,
-)
 from plone.restapi.controlpanels import RegistryConfigletPanel
+from redturtle.volto.interfaces import IRedTurtleVoltoSettings
+from redturtle.volto.interfaces import IRedTurtleVoltoSettingsControlpanel
 from zope.component import adapter
-from zope.interface import Interface
 from zope.interface import implementer
+from zope.interface import Interface
 
 
 @adapter(Interface, Interface)

src/redturtle/volto/restapi/services/querystringsearch/post.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
+from DateTime import DateTime
 from plone.app.event.base import get_events
 from plone.app.querystring import queryparser
 from plone.restapi.batching import HypermediaBatch
@@ -8,10 +9,10 @@
 from plone.restapi.interfaces import ISerializeToJsonSummary
 from plone.restapi.services.querystringsearch.get import QuerystringSearchPost
 from zope.component import getMultiAdapter
-from DateTime import DateTime
 
 import logging
 
+
 logger = logging.getLogger(__name__)