System Message: WARNING/2 (<string>, line 136)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from plone.restapi.interfaces import ISerializeToJson
    from zope.component import getMultiAdapter

    serializer = getMultiAdapter((obj, request), ISerializeToJson)
    data = serializer(include_items=False)

System Message: WARNING/2 (<string>, line 149)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from plone.restapi.interfaces import IDeserializeFromJson
    from zope.component import getMultiAdapter

    container.invokeFactory(item['@type'], item['id'])
    deserializer = getMultiAdapter((new, self.request), IDeserializeFromJson)
    new = deserializer(validate_all=False, data=item)

System Message: WARNING/2 (<string>, line 262)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collective.exportimport.export_content import ExportContent

    class CustomExportContent(ExportContent):

        QUERY = {
            'Document': {'review_state': ['published', 'pending']},
        }

        DROP_PATHS = [
            '/Plone/userportal',
            '/Plone/en/obsolete_content',
        ]

        DROP_UIDS = [
            '71e3e0a6f06942fea36536fbed0f6c42',
        ]

        def update(self):
            """Use this to override stuff before the export starts
            (e.g. force a specific language in the request)."""

        def start(self):
            """Hook to do something before export."""

        def finish(self):
            """Hook to do something after export."""

        def global_obj_hook(self, obj):
            """Inspect the content item before serialisation data.
            Bad: Changing the content-item is a horrible idea.
            Good: Return None if you want to skip this particular object.
            """
            return obj

        def global_dict_hook(self, item, obj):
            """Use this to modify or skip the serialized data.
            Return None if you want to skip this particular object.
            """
            return item

        def dict_hook_document(self, item, obj):
            """Use this to modify or skip the serialized data by type.
            Return the modified dict (item) or None if you want to skip this particular object.
            """
            return item

System Message: WARNING/2 (<string>, line 313)

Cannot analyze code. Pygments package not found.

.. code-block:: xml

  <browser:page
      name="export_content"
      for="zope.interface.Interface"
      class=".custom_export.CustomExportContent"
      layer="My.Custom.IBrowserlayer"
      permission="cmf.ManagePortal"
      />

System Message: WARNING/2 (<string>, line 327)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collective.exportimport.import_content import ImportContent

    class CustomImportContent(ImportContent):

        CONTAINER = {'Event': '/imported-events'}

        # These fields will be ignored
        DROP_FIELDS = ['relatedItems']

        # Items with these uid will be ignored
        DROP_UIDS = ['04d1477583c74552a7fcd81a9085c620']

        # These paths will be ignored
        DROP_PATHS = ['/Plone/doormat/', '/Plone/import_files/']

        # Default values for some fields
        DEFAULTS = {'which_price': 'normal'}

        def start(self):
            """Hook to do something before importing one file."""

        def finish(self):
            """Hook to do something after importing one file."""

        def global_dict_hook(self, item):
            if isinstance(item.get('description', None), dict):
                item['description'] = item['description']['data']
            if isinstance(item.get('rights', None), dict):
                item['rights'] = item['rights']['data']
            return item

        def dict_hook_customtype(self, item):
            # change the type
            item['@type'] = 'anothertype'
            # drop a field
            item.pop('experiences', None)
            return item

        def handle_file_container(self, item):
            """Use this to specify the container in which to create the item in.
            Return the container for this particular object.
            """
            return return self.portal['imported_files']

System Message: WARNING/2 (<string>, line 375)

Cannot analyze code. Pygments package not found.

.. code-block:: xml

  <browser:page
      name="import_content"
      for="zope.interface.Interface"
      class=".custom_import.CustomImportContent"
      layer="My.Custom.IBrowserlayer"
      permission="cmf.ManagePortal"
      />

System Message: WARNING/2 (<string>, line 391)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from plone import api
    from Products.Five import BrowserView

    class ExportAll(BrowserView):

        def __call__(self):
            export_content = api.content.get_view("export_content", self.context, self.request)
            self.request.form["form.submitted"] = True
            export_content(
                portal_type=["Folder", "Document", "News Item", "File", "Image"],  # only export these
                include_blobs=2,  # Export files and images as blob paths
                download_to_server=True)

            other_exports = [
                "export_relations",
                "export_members",
                "export_translations",
                "export_localroles",
                "export_ordering",
                "export_defaultpages",
                "export_discussion",
                "export_portlets",
                "export_redirects",
            ]
            for name in other_exports:
                view = api.content.get_view(name, portal, request)
                # This saves each export in var/instance/export_xxx.json
                view(download_to_server=True)

            # Important! Redirect to prevent infinite export loop :)
            return self.request.response.redirect(self.context.absolute_url())

System Message: WARNING/2 (<string>, line 427)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collective.exportimport.fix_html import fix_html_in_content_fields
    from collective.exportimport.fix_html import fix_html_in_portlets
    from pathlib import Path
    from plone import api
    from Products.Five import BrowserView


    class ImportAll(BrowserView):

        def __call__(self):
            portal = api.portal.get()

            # Import content
            view = api.content.get_view("import_content", portal, request)
            request.form["form.submitted"] = True
            request.form["commit"] = 500
            view(server_file="Plone.json", return_json=True)
            transaction.commit()

            # Run all other imports
            other_imports = [
                "relations",
                "members",
                "translations",
                "localroles",
                "ordering",
                "defaultpages",
                "discussion",
                "portlets",
                "redirects",
            ]
            cfg = getConfiguration()
            directory = Path(cfg.clienthome) / "import"
            for name in other_imports:
                view = api.content.get_view(f"import_{name}", portal, request)
                path = Path(directory) / f"export_{name}.json"
                results = view(jsonfile=path.read_text(), return_json=True)
                logger.info(results)
                transaction.commit()

            # Run cleanup steps
            results = fix_html_in_content_fields()
            logger.info("Fixed html for %s content items", results)
            transaction.commit()

            results = fix_html_in_portlets()
            logger.info("Fixed html for %s portlets", results)
            transaction.commit()

            reset_dates = api.content.get_view("reset_dates", portal, request)
            reset_dates()
            transaction.commit()

System Message: WARNING/2 (<string>, line 496)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_obj_hook(self, obj):
        # Drop subtopics
        if obj.portal_type == "Topic" and obj.__parent__.portal_type == "Topic":
            return

        # Drop files and images from PFG formfolders
        if obj.__parent__.portal_type == "FormFolder":
            return
        return obj

System Message: WARNING/2 (<string>, line 525)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_dict_hook(self, item, obj):
        if obj.isPrincipiaFolderish and ".wf_policy_config" in obj.keys():
            wf_policy = obj[".wf_policy_config"]
            item["exportimport.workflow_policy"] = {
                "workflow_policy_below": wf_policy.workflow_policy_below,
                "workflow_policy_in": wf_policy.workflow_policy_in,
            }
        return item

System Message: WARNING/2 (<string>, line 538)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_obj_hook(self, obj, item):
        wf_policy = item.get("exportimport.workflow_policy")
        if wf_policy:
            obj.manage_addProduct["CMFPlacefulWorkflow"].manage_addWorkflowPolicyConfig()
            wf_policy_config = obj[".wf_policy_config"]
            wf_policy_config.setPolicyIn(wf_policy["workflow_policy_in"], update_security=True)
            wf_policy_config.setPolicyBelow(wf_policy["workflow_policy_below"], update_security=True)

System Message: WARNING/2 (<string>, line 556)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_dict_hook(self, item):
        effective = item.get('effective', None)
        expires = item.get('expires', None)
        if effective and expires and expires <= effective:
            item.pop('expires')
        return item

System Message: WARNING/2 (<string>, line 567)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_dict_hook(self, item):
        item["creators"] = [i for i in item.get("creators", []) if i]
        return item

System Message: WARNING/2 (<string>, line 576)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def dict_hook_helpcenter(self, item):
        item["@type"] = "Folder"
        item["layout"] = "listing_view"
        return item

    def dict_hook_helpcenterglossary(self, item):
        item["@type"] = "Folder"
        item["layout"] = "listing_view"
        return item

    def dict_hook_helpcenterinstructionalvideo(self, item):
        item["@type"] = "File"
        if item.get("video_file"):
            item["file"] = item["video_file"]
        return item

    def dict_hook_helpcenterlink(self, item):
        item["@type"] = "Link"
        item["remoteUrl"] = item.get("url", None)
        return item

    def dict_hook_helpcenterreferencemanualpage(self, item):
        item["@type"] = "Document"
        return item

System Message: WARNING/2 (<string>, line 606)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    PORTAL_TYPE_MAPPING = {
        "Topic": "Collection",
        "FormFolder": "EasyForm",
        "HelpCenter": "Folder",
    }

    def global_dict_hook(self, item):
        if item.get("exportimport.constrains"):
            types_fixed = []
            for portal_type in item["exportimport.constrains"]["locally_allowed_types"]:
                if portal_type in PORTAL_TYPE_MAPPING:
                    types_fixed.append(PORTAL_TYPE_MAPPING[portal_type])
                elif portal_type in ALLOWED_TYPES:
                    types_fixed.append(portal_type)
            item["exportimport.constrains"]["locally_allowed_types"] = list(set(types_fixed))

            types_fixed = []
            for portal_type in item["exportimport.constrains"]["immediately_addable_types"]:
                if portal_type in PORTAL_TYPE_MAPPING:
                    types_fixed.append(PORTAL_TYPE_MAPPING[portal_type])
                elif portal_type in ALLOWED_TYPES:
                    types_fixed.append(portal_type)
            item["exportimport.constrains"]["immediately_addable_types"] = list(set(types_fixed))
        return item

System Message: WARNING/2 (<string>, line 637)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    REVIEW_STATE_MAPPING = {
        "internal": "published",
        "internally_published": "published",
        "obsolete": "private",
        "hidden": "private",
    }

    def global_dict_hook(self, item):
        if item.get("review_state") in REVIEW_STATE_MAPPING:
            item["review_state"] = REVIEW_STATE_MAPPING[item["review_state"]]
        return item

System Message: WARNING/2 (<string>, line 661)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from zope.annotation.interfaces import IAnnotations
    ANNOTATIONS_TO_EXPORT = [
        "syndication_settings",
    ]
    ANNOTATIONS_KEY = 'exportimport.annotations'

    class CustomExportContent(ExportContent):

        def global_dict_hook(self, item, obj):
            item = self.export_annotations(item, obj)
            return item

        def export_annotations(self, item, obj):
            results = {}
            annotations = IAnnotations(obj)
            for key in ANNOTATIONS_TO_EXPORT:
                data = annotations.get(key)
                if data:
                    results[key] = IJsonCompatible(data, None)
            if results:
                item[ANNOTATIONS_KEY] = results
            return item

System Message: WARNING/2 (<string>, line 688)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from zope.annotation.interfaces import IAnnotations
    ANNOTATIONS_KEY = "exportimport.annotations"

    class CustomImportContent(ImportContent):

        def global_obj_hook(self, obj, item):
            item = self.import_annotations(obj, item)
            return item

        def import_annotations(self, obj, item):
            annotations = IAnnotations(obj)
            for key in item.get(ANNOTATIONS_KEY, []):
                annotations[key] = item[ANNOTATIONS_KEY][key]
            return item

System Message: WARNING/2 (<string>, line 715)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from zope.interface import directlyProvidedBy

    MARKER_INTERFACES_TO_EXPORT = [
        "collective.easyslider.interfaces.ISliderPage",
        "plone.app.layout.navigation.interfaces.INavigationRoot",
    ]
    MARKER_INTERFACES_KEY = "exportimport.marker_interfaces"

    class CustomExportContent(ExportContent)

        def global_dict_hook(self, item, obj):
            item = self.export_marker_interfaces(item, obj)
            return item

        def export_marker_interfaces(self, item, obj):
            interfaces = [i.__identifier__ for i in directlyProvidedBy(obj)]
            interfaces = [i for i in interfaces if i in MARKER_INTERFACES_TO_EXPORT]
            if interfaces:
                item[MARKER_INTERFACES_KEY] = interfaces
            return item

System Message: WARNING/2 (<string>, line 740)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from plone.dexterity.utils import resolveDottedName
    from zope.interface import alsoProvides

    MARKER_INTERFACES_KEY = "exportimport.marker_interfaces"

    class CustomImportContent(ImportContent):

        def global_obj_hook_before_deserializing(self, obj, item):
            """Apply marker interfaces before deserializing."""
            for iface_name in item.pop(MARKER_INTERFACES_KEY, []):
                try:
                    iface = resolveDottedName(iface_name)
                    if not iface.providedBy(obj):
                        alsoProvides(obj, iface)
                        logger.info("Applied marker interface %s to %s", iface_name, obj.absolute_url())
                except ModuleNotFoundError:
                    pass
            return obj, item

System Message: WARNING/2 (<string>, line 767)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    VERSIONED_TYPES = [
        "Document",
        "News Item",
        "Event",
        "Link",
    ]

    def start(self):
        self.items_without_parent = []
        portal_types = api.portal.get_tool("portal_types")
        for portal_type in VERSIONED_TYPES:
            fti = portal_types.get(portal_type)
            behaviors = list(fti.behaviors)
            if 'plone.versioning' in behaviors:
                logger.info(f"Disable versioning for {portal_type}")
                behaviors.remove('plone.versioning')
            fti.behaviors = behaviors

System Message: WARNING/2 (<string>, line 789)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from Products.CMFEditions.interfaces.IModifier import FileTooLargeToVersionError

    VERSIONED_TYPES = [
        "Document",
        "News Item",
        "Event",
        "Link",
    ]

    class ImportAll(BrowserView):

        # re-enable versioning
        portal_types = api.portal.get_tool("portal_types")
        for portal_type in VERSIONED_TYPES:
            fti = portal_types.get(portal_type)
            behaviors = list(fti.behaviors)
            if "plone.versioning" not in behaviors:
                behaviors.append("plone.versioning")
                logger.info(f"Enable versioning for {portal_type}")
            if "plone.locking" not in behaviors:
                behaviors.append("plone.locking")
                logger.info(f"Enable locking for {portal_type}")
            fti.behaviors = behaviors
        transaction.get().note("Re-enabled versioning")
        transaction.commit()

        # create initial version for all versioned types
        logger.info("Creating initial versions")
        portal_repository = api.portal.get_tool("portal_repository")
        brains = api.content.find(portal_type=VERSIONED_TYPES)
        total = len(brains)
        for index, brain in enumerate(brains):
            obj = brain.getObject()
            try:
                portal_repository.save(obj=obj, comment="Imported Version")
            except FileTooLargeToVersionError:
                pass
            if not index % 1000:
                msg = f"Created versions for {index} of {total} items."
                logger.info(msg)
                transaction.get().note(msg)
                transaction.commit()
        msg = "Created initial versions"
        transaction.get().note(msg)
        transaction.commit()

System Message: WARNING/2 (<string>, line 1007)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_dict_hook(self, item, obj):
        # Find language of the nearest parent with a language
        # Usefull for LinguaPlone sites where some content is languageindependent
        parent = obj.__parent__
        for ancestor in parent.aq_chain:
            if IPloneSiteRoot.providedBy(ancestor):
                # keep language for root content
                nearest_ancestor_lang = item["language"]
                break
            if getattr(ancestor, "getLanguage", None) and ancestor.getLanguage():
                nearest_ancestor_lang = ancestor.getLanguage()
                item["parent"]["language"] = nearest_ancestor_lang
                break

        # This forces "wrong" languages to the nearest parents language
        if "language" in item and item["language"] != nearest_ancestor_lang:
            logger.info(u"Forcing %s (was %s) for %s %s ", nearest_ancestor_lang, item["language"], item["@type"], item["@id"])
            item["language"] = nearest_ancestor_lang

        # set missing language
        if not item.get("language"):
            item["language"] = nearest_ancestor_lang

        # add info on translations to help find the right container
        # usually this idone by export_translations
        # but when migrating from LP to pam you sometimes want to check the
        # tranlation info during import
        if getattr(obj.aq_base, "getTranslations", None) is not None:
            translations = obj.getTranslations()
            if translations:
                item["translation"] = {}
                for lang in translations:
                    uuid = IUUID(translations[lang][0], None)
                    if uuid == item["UID"]:
                        continue
                    translation = translations[lang][0]
                    if not lang:
                        lang = "no_language"
                    item["translation"][lang] = translation.absolute_url()

System Message: WARNING/2 (<string>, line 1051)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def global_dict_hook(self, item):

        # Adapt this to your site
        languages = ["en", "fr", "de"]
        default_language = "en"
        portal_id = "Plone"

        # No language => lang of parent or default
        if item.get("language") not in languages:
            if item["parent"].get("language"):
                item["language"] = item["parent"]["language"]
            else:
                item["language"] = default_language

        lang = item["language"]

        if item["parent"].get("language") != item["language"]:
            logger.debug(f"Inconsistent lang: item is {lang}, parent is {item['parent'].get('language')} for {item['@id']}")

        # Move item to the correct language-root-folder
        # This is only relevant for items in the site-root.
        # Most items containers are usually looked up by the uuid of the old parent
        url = item["@id"]
        parent_url = item["parent"]["@id"]

        url = url.replace(f"/{portal_id}/", f"/{portal_id}/{lang}/", 1)
        parent_url = parent_url.replace(f"/{portal_id}", f"/{portal_id}/{lang}", 1)

        item["@id"] = url
        item["parent"]["@id"] = parent_url

        return item

System Message: WARNING/2 (<string>, line 1091)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def update(self):
        self.items_without_parent = []

    def create_container(self, item):
        # Override create_container to never create parents
        self.items_without_parent.append(item)

    def finish(self):
        # export content without parents
        if self.items_without_parent:
            data = json.dumps(self.items_without_parent, sort_keys=True, indent=4)
            number = len(self.items_without_parent)
            cfg = getConfiguration()
            filename = 'content_without_parent.json'
            filepath = os.path.join(cfg.clienthome, filename)
            with open(filepath, 'w') as f:
                f.write(data)
            msg = u"Saved {} items without parent to {}".format(number, filepath)
            logger.info(msg)
            api.portal.show_message(msg, self.request)

System Message: WARNING/2 (<string>, line 1123)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collective.exportimport.export_other import BaseExport
    from plone import api

    import six

    class ExportZopeUsers(BaseExport):

        AUTO_ROLES = ["Authenticated"]

        def __call__(self, download_to_server=False):
            self.title = "Export Zope users"
            self.download_to_server = download_to_server
            portal = api.portal.get()
            app = portal.__parent__
            self.acl = app.acl_users
            self.pms = api.portal.get_tool("portal_membership")
            data = self.all_zope_users()
            self.download(data)

        def all_zope_users(self):
            results = []
            for user in self.acl.searchUsers():
                data = self._getUserData(user["userid"])
                data['title'] = user['title']
                results.append(data)
            return results

        def _getUserData(self, userId):
            member = self.pms.getMemberById(userId)
            roles = [
                role
                for role in member.getRoles()
                if role not in self.AUTO_ROLES
            ]
            # userid, password, roles
            props = {
                "username": userId,
                "password": json_compatible(self._getUserPassword(userId)),
                "roles": json_compatible(roles),
            }
            return props

        def _getUserPassword(self, userId):
            users = self.acl.users
            passwords = users._user_passwords
            password = passwords.get(userId, "")
            return password

System Message: WARNING/2 (<string>, line 1175)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    class ImportZopeUsers(BrowserView):

        def __call__(self, jsonfile=None, return_json=False):
            if jsonfile:
                self.portal = api.portal.get()
                status = "success"
                try:
                    if isinstance(jsonfile, str):
                        return_json = True
                        data = json.loads(jsonfile)
                    elif isinstance(jsonfile, FileUpload):
                        data = json.loads(jsonfile.read())
                    else:
                        raise ("Data is neither text nor upload.")
                except Exception as e:
                    status = "error"
                    logger.error(e)
                    api.portal.show_message(
                        u"Failure while uploading: {}".format(e),
                        request=self.request,
                    )
                else:
                    members = self.import_members(data)
                    msg = u"Imported {} members".format(members)
                    api.portal.show_message(msg, self.request)
                if return_json:
                    msg = {"state": status, "msg": msg}
                    return json.dumps(msg)

            return self.index()

        def import_members(self, data):
            app = self.portal.__parent__
            acl = app.acl_users
            counter = 0
            for item in data:
                username = item["username"]
                password = item.pop("password")
                roles = item.pop("roles", [])
                if not username or not password or not roles:
                    continue
                title = item.pop("title", None)
                acl.users.addUser(username, title, password)
                for role in roles:
                    acl.roles.assignRoleToPrincipal(role, username)
                counter += 1
            return counter

System Message: WARNING/2 (<string>, line 1371)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def dict_hook_formfolder(self, item, obj):
        item["@type"] = "EasyForm"
        item["is_folderish"] = False

        from collective.easyform.migration.fields import fields_model
        from collective.easyform.migration.actions import actions_model

        # this does most of the heavy lifting...
        item["fields_model"] = fields_model(obj)
        item["actions_model"] = actions_model(obj)

        # handle thankspage
        pfg_thankspage = obj.get(obj.getThanksPage(), None)
        if pfg_thankspage:
            item["thankstitle"] = pfg_thankspage.title
            item["thanksdescription"] = pfg_thankspage.Description()
            item["showAll"] = pfg_thankspage.showAll
            item["showFields"] = pfg_thankspage.showFields
            item["includeEmpties"] = pfg_thankspage.includeEmpties
            item["thanksPrologue"] = json_compatible(pfg_thankspage.thanksPrologue.raw)
            item["thanksEpilogue"] = json_compatible(pfg_thankspage.thanksEpilogue.raw)

        # optional
        item["exportimport._inputStorage"] = self.export_saved_data(obj)

        # Drop some PFG fields no longer needed
        obsolete_fields = [
            "layout",
            "actionAdapter",
            "checkAuthenticator",
            "constrainTypesMode",
            "location",
            "thanksPage",
        ]
        for key in obsolete_fields:
            item.pop(key, None)

        # optional: disable tabs for imported forms
        item["form_tabbing"] = False

        # fix some custom validators
        replace_mapping = {
            "request.form['": "request.form['form.widgets.",
            "request.form.get('": "request.form.get('form.widgets.",
            "member and member.id or ''": "member and member.getProperty('id', '') or ''",
        }

        # fix overrides in actions and fields to use form.widgets.xyz instead of xyz
        for schema in ["actions_model", "fields_model"]:
            for old, new in replace_mapping.items():
                if old in item[schema]:
                    item[schema] = item[schema].replace(old, new)

            # add your own fields if you have these issues...
            for fieldname in [
                "email",
                "replyto",
            ]:
                if "request/form/{}".format(fieldname) in item[schema]:
                    item[schema] = item[schema].replace("request/form/{}".format(fieldname), "python: request.form.get('form.widgets.{}')".format(fieldname))

        return item

    def export_saved_data(self, obj):
        actions = {}
        for data_adapter in obj.objectValues("FormSaveDataAdapter"):
            data_adapter_name = data_adapter.getId()
            actions[data_adapter_name] = {}
            cols = data_adapter.getColumnNames()
            column_count_mismatch = False
            for idx, row in enumerate(data_adapter.getSavedFormInput()):
                if len(row) != len(cols):
                    column_count_mismatch = True
                    logger.debug("Column count mismatch at row %s", idx)
                    continue
                data = {}
                for key, value in zip(cols, row):
                    data[key] = json_compatible(value)
                id_ = int(time() * 1000)
                while id_ in actions[data_adapter_name]:  # avoid collisions during export
                    id_ += 1
                data["id"] = id_
                actions[data_adapter_name][id_] = data
            if column_count_mismatch:
                logger.info(
                    "Number of columns does not match for all rows. Some data were skipped in "
                    "data adapter %s/%s",
                    "/".join(obj.getPhysicalPath()),
                    data_adapter_name,
                )
        return actions

System Message: WARNING/2 (<string>, line 1467)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    def obj_hook_easyform(self, obj, item):
        if not item.get("exportimport._inputStorage"):
            return
        from collective.easyform.actions import SavedDataBTree
        from persistent.mapping import PersistentMapping
        if not hasattr(obj, '_inputStorage'):
            obj._inputStorage = PersistentMapping()
        for name, data in item["exportimport._inputStorage"].items():
            obj._inputStorage[name] = SavedDataBTree()
            for key, row in data.items():
                 obj._inputStorage[name][int(key)] = row

System Message: WARNING/2 (<string>, line 1487)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collective.exportimport.serializer import get_dx_blob_path
    from plone.app.textfield.value import RichTextValue
    from plone.namedfile.file import NamedBlobImage
    from plone.restapi.interfaces import IJsonCompatible
    from z3c.relationfield import RelationValue
    from zope.annotation.interfaces import IAnnotations

    def global_dict_hook(self, item, obj):
        item = self.handle_cover(item, obj)
        return item

    def handle_cover(self, item, obj):
        if ICover.providedBy(obj):
            item['tiles'] = {}
            annotations = IAnnotations(obj)
            for tile in obj.get_tiles():
                annotation_key = 'plone.tiles.data.{}'.format(tile['id'])
                annotation = annotations.get(annotation_key, None)
                if annotation is None:
                    continue
                tile_data = self.serialize_tile(annotation)
                tile_data['type'] = tile['type']
                item['tiles'][tile['id']] = tile_data
        return item

    def serialize_tile(self, annotation):
        data = {}
        for key, value in annotation.items():
            if isinstance(value, RichTextValue):
                value = value.raw
            elif isinstance(value, RelationValue):
                value = value.to_object.UID()
            elif isinstance(value, NamedBlobImage):
                blobfilepath = get_dx_blob_path(value)
                if not blobfilepath:
                    continue
                value = {
                    "filename": value.filename,
                    "content-type": value.contentType,
                    "size": value.getSize(),
                    "blob_path": blobfilepath,
                }
            data[key] = IJsonCompatible(value, None)
        return data

System Message: WARNING/2 (<string>, line 1536)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from collections import defaultdict
    from collective.exportimport.import_content import get_absolute_blob_path
    from plone.app.textfield.interfaces import IRichText
    from plone.app.textfield.interfaces import IRichTextValue
    from plone.namedfile.file import NamedBlobImage
    from plone.namedfile.interfaces import INamedBlobImageField
    from plone.tiles.interfaces import ITileType
    from zope.annotation.interfaces import IAnnotations
    from zope.component import getUtilitiesFor
    from zope.schema import getFieldsInOrder

    COVER_CONTENT = [
        "collective.cover.content",
    ]

    def global_obj_hook(self, obj, item):
        if item["@type"] in COVER_CONTENT and "tiles" in item:
            item = self.import_tiles(obj, item)

    def import_tiles(self, obj, item):
        RICHTEXT_TILES = defaultdict(list)
        IMAGE_TILES = defaultdict(list)
        for tile_name, tile_type in getUtilitiesFor(ITileType):
            for fieldname, field in getFieldsInOrder(tile_type.schema):
                if IRichText.providedBy(field):
                    RICHTEXT_TILES[tile_name].append(fieldname)
                if INamedBlobImageField.providedBy(field):
                    IMAGE_TILES[tile_name].append(fieldname)

        annotations = IAnnotations(obj)
        prefix = "plone.tiles.data."
        for uid, tile in item["tiles"].items():
            # TODO: Maybe create all tiles that do not need to be defferred?
            key = prefix + uid
            tile_name = tile.pop("type", None)
            # first set raw data
            annotations[key] = item["tiles"][uid]
            for fieldname in RICHTEXT_TILES.get(tile_name, []):
                raw = annotations[key][fieldname]
                if raw is not None and not IRichTextValue.providedBy(raw):
                    annotations[key][fieldname] = RichTextValue(raw, "text/html", "text/x-html-safe")
            for fieldname in IMAGE_TILES.get(tile_name, []):
                data = annotations[key][fieldname]
                if data is not None:
                    blob_path = data.get("blob_path")
                    if not blob_path:
                        continue

                    abs_blob_path = get_absolute_blob_path(obj, blob_path)
                    if not abs_blob_path:
                        logger.info("Blob path %s for tile %s of %s %s does not exist!", blob_path, tile, obj.portal_type, obj.absolute_url())
                        continue
                    # Determine the class to use: file or image.
                    filename = data["filename"]
                    content_type = data["content-type"]

                    # Write the field.
                    with open(abs_blob_path, "rb") as myfile:
                        blobdata = myfile.read()
                    image = NamedBlobImage(
                        data=blobdata,
                        contentType=content_type,
                        filename=filename,
                    )
                    annotations[key][fieldname] = image
        return item

System Message: WARNING/2 (<string>, line 1614)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    class CustomImportContent(ImportContent):

        def global_dict_hook(self, item):
            if item["@type"] in ["Collection", "Topic"]:
                item = self.fix_query(item)

        def fix_query(self, item):
            item["@type"] = "Collection"
            query = item.pop("query", [])
            if not query:
                logger.info("Drop item without query: %s", item["@id"])
                return

            fixed_query = []
            indexes_to_fix = [
                "portal_type",
                "review_state",
                "Creator",
                "Subject",
            ]
            operator_mapping = {
                # old -> new
                "plone.app.querystring.operation.selection.is":
                    "plone.app.querystring.operation.selection.any",
                "plone.app.querystring.operation.string.is":
                    "plone.app.querystring.operation.selection.any",
            }

            for crit in query:
                if crit["i"] == "portal_type" and len(crit["v"]) > 30:
                    # Criterion is all types
                    continue

                if crit["o"].endswith("relativePath") and crit["v"] == "..":
                    # relativePath no longer accepts ..
                    crit["v"] = "..::1"

                if crit["i"] in indexes_to_fix:
                    for old_operator, new_operator in operator_mapping.items():
                        if crit["o"] == old_operator:
                            crit["o"] = new_operator

                if crit["i"] == "portal_type":
                    # Some types may have changed their names
                    fixed_types = []
                    for portal_type in crit["v"]:
                        fixed_type = PORTAL_TYPE_MAPPING.get(portal_type, portal_type)
                        fixed_types.append(fixed_type)
                    crit["v"] = list(set(fixed_types))

                if crit["i"] == "review_state":
                    # Review states may have changed their names
                    fixed_states = []
                    for review_state in crit["v"]:
                        fixed_state = REVIEW_STATE_MAPPING.get(review_state, review_state)
                        fixed_states.append(fixed_state)
                    crit["v"] = list(set(fixed_states))

                if crit["o"] == "plone.app.querystring.operation.string.currentUser":
                    crit["v"] = ""

                fixed_query.append(crit)
            item["query"] = fixed_query

            if not item["query"]:
                logger.info("Drop collection without query: %s", item["@id"])
                return
            return item

System Message: WARNING/2 (<string>, line 1697)

Cannot analyze code. Pygments package not found.

.. code-block:: python

    from App.config import getConfiguration
    from bs4 import BeautifulSoup
    from collective.exportimport.fix_html import fix_html_in_content_fields
    from collective.exportimport.fix_html import fix_html_in_portlets
    from contentimport.interfaces import IContentimportLayer
    from logging import getLogger
    from pathlib import Path
    from plone import api
    from plone.volto.browser.migrate_to_volto import migrate_richtext_to_blocks
    from plone.volto.setuphandlers import add_behavior
    from plone.volto.setuphandlers import remove_behavior
    from Products.CMFPlone.utils import get_installer
    from Products.Five import BrowserView
    from zope.interface import alsoProvides

    import requests
    import transaction

    logger = getLogger(__name__)

    DEFAULT_ADDONS = []


    class ImportAll(BrowserView):

        def __call__(self):

            request = self.request

            # Check if Blocks-conversion-tool is running
            headers = {
                "Accept": "application/json",
                "Content-Type": "application/json",
            }
            r = requests.post(
                "http://localhost:5000/html", headers=headers, json={"html": "<p>text</p>"}
            )
            r.raise_for_status()

            # Submit a simple form template to trigger the import
            if not request.form.get("form.submitted", False):
                return self.index()

            portal = api.portal.get()
            alsoProvides(request, IContentimportLayer)

            installer = get_installer(portal)
            if not installer.is_product_installed("contentimport"):
                installer.install_product("contentimport")

            # install required addons
            for addon in DEFAULT_ADDONS:
                if not installer.is_product_installed(addon):
                    installer.install_product(addon)

            # Fake the target being a classic site even though plone.volto is installed...
            # 1. Allow Folders and Collections (they are disabled in Volto by default)
            portal_types = api.portal.get_tool("portal_types")
            portal_types["Collection"].global_allow = True
            portal_types["Folder"].global_allow = True
            # 2. Enable richtext behavior (otherwise no text will be imported)
            for type_ in ["Document", "News Item", "Event"]:
                add_behavior(type_, "plone.richtext")

            transaction.commit()
            cfg = getConfiguration()
            directory = Path(cfg.clienthome) / "import"

            # Import content
            view = api.content.get_view("import_content", portal, request)
            request.form["form.submitted"] = True
            request.form["commit"] = 500
            view(server_file="Plone.json", return_json=True)
            transaction.commit()

            # Run all other imports
            other_imports = [
                "relations",
                "members",
                "translations",
                "localroles",
                "ordering",
                "defaultpages",
                "discussion",
                "portlets",  # not really useful in Volto
                "redirects",
            ]
            for name in other_imports:
                view = api.content.get_view(f"import_{name}", portal, request)
                path = Path(directory) / f"export_{name}.json"
                if path.exists():
                    results = view(jsonfile=path.read_text(), return_json=True)
                    logger.info(results)
                    transaction.get().note(f"Finished import_{name}")
                    transaction.commit()
                else:
                    logger.info(f"Missing file: {path}")

            # Optional: Run html-fixers on richtext
            fixers = [anchor_fixer]
            results = fix_html_in_content_fields(fixers=fixers)
            msg = "Fixed html for {} content items".format(results)
            logger.info(msg)
            transaction.get().note(msg)
            transaction.commit()

            results = fix_html_in_portlets()
            msg = "Fixed html for {} portlets".format(results)
            logger.info(msg)
            transaction.get().note(msg)
            transaction.commit()

            view = api.content.get_view("updateLinkIntegrityInformation", portal, request)
            results = view.update()
            msg = f"Updated linkintegrity for {results} items"
            logger.info(msg)
            transaction.get().note(msg)
            transaction.commit()

            # Rebuilding the catalog is necessary to prevent issues later on
            catalog = api.portal.get_tool("portal_catalog")
            logger.info("Rebuilding catalog...")
            catalog.clearFindAndRebuild()
            msg = "Finished rebuilding catalog!"
            logger.info(msg)
            transaction.get().note(msg)
            transaction.commit()

            # This uses the blocks-conversion-tool to migrate to blocks
            logger.info("Start migrating richtext to blocks...")
            migrate_richtext_to_blocks()
            msg = "Finished migrating richtext to blocks"
            transaction.get().note(msg)
            transaction.commit()

            # Reuse the migration-form from plon.volto to do some more tasks
            view = api.content.get_view("migrate_to_volto", portal, request)
            # Yes, wen want to migrate default pages
            view.migrate_default_pages = True
            view.slate = True
            logger.info("Start migrating Folders to Documents...")
            view.do_migrate_folders()
            msg = "Finished migrating Folders to Documents!"
            transaction.get().note(msg)
            transaction.commit()

            logger.info("Start migrating Collections to Documents...")
            view.migrate_collections()
            msg = "Finished migrating Collections to Documents!"
            transaction.get().note(msg)
            transaction.commit()

            reset_dates = api.content.get_view("reset_dates", portal, request)
            reset_dates()
            transaction.commit()

            # Disallow folders and collections again
            portal_types["Collection"].global_allow = False
            portal_types["Folder"].global_allow = False

            # Disable richtext behavior again
            for type_ in ["Document", "News Item", "Event"]:
                remove_behavior(type_, "plone.richtext")

            return request.response.redirect(portal.absolute_url())


    def anchor_fixer(text, obj=None):
        """Remove anchors since they are not supported by Volto yet"""
        soup = BeautifulSoup(text, "html.parser")
        for link in soup.find_all("a"):
            if not link.get("href") and not link.text:
                # drop empty links (e.g. anchors)
                link.decompose()
            elif not link.get("href") and link.text:
                # drop links without a href but keep the text
                link.unwrap()
        return soup.decode()