Skip to content

Commit

Permalink
fix remove anchors macro to stop it removing comments on embeds
Browse files Browse the repository at this point in the history
  • Loading branch information
@marwoodandrew
marwoodandrew committed Oct 25, 2024
1 parent 20a2655 commit d09c5a1
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 78 deletions.
19 changes: 10 additions & 9 deletions server/aap/macros/remove_anchors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,26 +24,27 @@ def remove_anchors(item, **kwargs):
"""

def clean_html(html):
cleaner = lxml.html.clean.Cleaner(remove_tags=['a'])
cleaner = lxml.html.clean.Cleaner(remove_tags=["a"])
cleaner.comments = False
root = lxml.html.fromstring(html)

for elem in root.iter():
elem.attrib.pop("id", None)
elem.attrib.pop("class", None)
if elem.tag in ('hl2', 'pre', 'note'):
elem.tag = 'p'
if elem.tag in ("hl2", "pre", "note"):
elem.tag = "p"

root = cleaner.clean_html(root)
return sd_etree.to_string(root, method="html")

if item.get('body_html'):
item['body_html'] = clean_html(item.get('body_html', ''))
if item.get("body_html"):
item["body_html"] = clean_html(item.get("body_html", ""))

return item


name = 'Remove Anchors'
label = 'Remove Links from text'
name = "Remove Anchors"
label = "Remove Links from text"
callback = remove_anchors
access_type = 'frontend'
action_type = 'direct'
access_type = "frontend"
action_type = "direct"
135 changes: 66 additions & 69 deletions server/aap/macros/remove_anchors_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,76 +22,73 @@ def test_ap_story_with_anchor(self):
"_id": "tag:localhost:2017:77b03a97-df04-446e-a112-94941f1bb12c",
"firstcreated": firstcreated,
"subject": [
{
"name": "lifestyle and leisure",
"qcode": "10000000"
},
{
"name": "politics",
"qcode": "11000000"
},
{
"name": "diplomacy",
"qcode": "11002000"
}
{"name": "lifestyle and leisure", "qcode": "10000000"},
{"name": "politics", "qcode": "11000000"},
{"name": "diplomacy", "qcode": "11002000"},
],
"place": [
{
"name": "United States"
}
],
'body_html': "<block>\n <p>UNITED NATIONS (AP) — Days after kicking out seven U.N. "
"officials, Ethiopia accused them without providing evidence Wednesday of inflating the "
"magnitude of humanitarian crisis and taking sides in the war in its Tigray region, while "
"U.N. Secretary-General Antonio Guterres pressed the country’s ambassador for documentation "
"of the allegations.</p>\n <p>The unexpected exchange came at a Security "
"Council meeting called to discuss the officials' expulsion amid what the U.N. sees as an "
"escalating humanitarian catastrophe in the Horn of Africa nation. To some council members, "
"the ejection of the officials — most of them with the U.N.'s humanitarian agency — will "
"complicate already difficult aid operations. </p>\n <p>Ethiopian "
"Ambassador Taye Atske Selassie laid out newly detailed claims about the officials. He "
"alleged they inflated the number of needy people by 1 million, cheered the Tigrayan "
"forces who are fighting the government, invented a dozen deaths in a camp for displaced "
"people, and helped channel Ethiopian migrants from Saudi Arabia to another African nation "
"“for training and preparation” to fight with the Tigrayans, among other a"
"ccusations. </p>\n <p>“Ethiopia deeply resents this experience,\" "
"the ambassador said, adding that the government had written to the U.N. about staff conduct "
"in July. </p>\n <p>A surprised Guterres responded that he had known "
"nothing of these allegations and that he had twice asked Prime Minister Abiy Ahmed to "
"send him details on any concerns about the impartiality of U.N. staffers.</p>\n "
" <p>Guterres, <a href=\"https://apnews.com/f5c9015472474462081a8fd769e5b18e\">who "
"maintains Ethiopia had no right under the U.N. charter to expel </a> the officials, took a "
"rare step for a secretary-general by responding directly in the council chamber. He asked "
"the ambassador to provide any written documents that the government may have about any "
"alleged wrongdoing by any of the seven officials. </p>\n <p>“It is my "
"duty to defend the honor of the United Nations,” Guterres told reporters afterward. He said "
"if such documents are provided, the U.N. will investigate why he wasn't alerted "
"about the matter. </p>\n <p>Ethiopia <a href=\"https://apnews.com/"
"article/africa-united-nations-ethiopia-tigray-4dbbff1c6fa65da8d9d5c4b2c0029125\">announced "
"the expulsions last Thursday</a>, accusing the U.N. officials of meddling in the country’s i"
"nternal business.</p>\n <p>The country’s foreign ministry later added "
"some more specific claims of “grave violations,” such as violating security agreements, "
"transferring communications equipment to be used by Tigray forces, spreading misinformation "
"and “politicization of humanitarian assistance.” </p>\n <p>But much of"
" what the ambassador said Wednesday had not been raised publicly before. </p>\n "
" <p>The expulsions came as the U.N. was increasingly outspoken about what it "
"calls the Ethiopian government’s de facto blockade of the Tigray region, where local forces"
" have been fighting government soldiers and allied troops since November. </p>\n "
" <p>The conflict began as a political dispute after Prime Minister Abiy Ahmed "
"sidelined the Tigrayan regional party that had dominated Ethiopia’s government for "
"decades. The clash spiraled into a war that has killed thousands of people and a "
"hunger crisis that threatens still more.</p>\n <p>Up to 7 million "
"people need food and other aid in Tigray and nearby regions where the fighting has sprea"
"d, and an estimated 400,000 people are living in “famine-like conditions,” Guterres said. "
"</p>\n <p>“The people of Ethiopia are suffering. We have no other "
"interest but to help stop that suffering,” he said. </p>\n <p>U.N. "
"humanitarian chief Martin Griffiths <a href=\"https://apnews.com/article/africa-health-"
"united-nations-only-on-ap-famine-a2b1639797c2a31973ce12985d82b865\">told The Associated "
"Press </a> last week that only 10% of needed humanitarian supplies have been reaching "
"Tigray in recent weeks, </p>\n <p>Five of the officials expelled work "
"with the U.N. humanitarian agency, another is with the U.N. human rights office and the "
"seventh is with UNICEF, the U.N. children’s agency.</p>"
"place": [{"name": "United States"}],
"body_html": "<block>\n <p>UNITED NATIONS (AP) — Days after kicking out seven U.N. "
"officials, Ethiopia accused them without providing evidence Wednesday of inflating the "
"magnitude of humanitarian crisis and taking sides in the war in its Tigray region, while "
"U.N. Secretary-General Antonio Guterres pressed the country’s ambassador for documentation "
"of the allegations.</p>\n <p>The unexpected exchange came at a Security "
"Council meeting called to discuss the officials' expulsion amid what the U.N. sees as an "
"escalating humanitarian catastrophe in the Horn of Africa nation. To some council members, "
"the ejection of the officials — most of them with the U.N.'s humanitarian agency — will "
"complicate already difficult aid operations. </p>\n <p>Ethiopian "
"Ambassador Taye Atske Selassie laid out newly detailed claims about the officials. He "
"alleged they inflated the number of needy people by 1 million, cheered the Tigrayan "
"forces who are fighting the government, invented a dozen deaths in a camp for displaced "
"people, and helped channel Ethiopian migrants from Saudi Arabia to another African nation "
"“for training and preparation” to fight with the Tigrayans, among other a"
'ccusations. </p>\n <p>“Ethiopia deeply resents this experience," '
"the ambassador said, adding that the government had written to the U.N. about staff conduct "
"in July. </p>\n <p>A surprised Guterres responded that he had known "
"nothing of these allegations and that he had twice asked Prime Minister Abiy Ahmed to "
"send him details on any concerns about the impartiality of U.N. staffers.</p>\n "
' <p>Guterres, <a href="https://apnews.com/f5c9015472474462081a8fd769e5b18e">who '
"maintains Ethiopia had no right under the U.N. charter to expel </a> the officials, took a "
"rare step for a secretary-general by responding directly in the council chamber. He asked "
"the ambassador to provide any written documents that the government may have about any "
"alleged wrongdoing by any of the seven officials. </p>\n <p>“It is my "
"duty to defend the honor of the United Nations,” Guterres told reporters afterward. He said "
"if such documents are provided, the U.N. will investigate why he wasn't alerted "
'about the matter. </p>\n <p>Ethiopia <a href="https://apnews.com/'
'article/africa-united-nations-ethiopia-tigray-4dbbff1c6fa65da8d9d5c4b2c0029125">announced '
"the expulsions last Thursday</a>, accusing the U.N. officials of meddling in the country’s i"
"nternal business.</p>\n <p>The country’s foreign ministry later added "
"some more specific claims of “grave violations,” such as violating security agreements, "
"transferring communications equipment to be used by Tigray forces, spreading misinformation "
"and “politicization of humanitarian assistance.” </p>\n <p>But much of"
" what the ambassador said Wednesday had not been raised publicly before. </p>\n "
" <p>The expulsions came as the U.N. was increasingly outspoken about what it "
"calls the Ethiopian government’s de facto blockade of the Tigray region, where local forces"
" have been fighting government soldiers and allied troops since November. </p>\n "
" <p>The conflict began as a political dispute after Prime Minister Abiy Ahmed "
"sidelined the Tigrayan regional party that had dominated Ethiopia’s government for "
"decades. The clash spiraled into a war that has killed thousands of people and a "
"hunger crisis that threatens still more.</p>\n <p>Up to 7 million "
"people need food and other aid in Tigray and nearby regions where the fighting has sprea"
"d, and an estimated 400,000 people are living in “famine-like conditions,” Guterres said. "
"</p>\n <p>“The people of Ethiopia are suffering. We have no other "
"interest but to help stop that suffering,” he said. </p>\n <p>U.N. "
'humanitarian chief Martin Griffiths <a href="https://apnews.com/article/africa-health-'
'united-nations-only-on-ap-famine-a2b1639797c2a31973ce12985d82b865">told The Associated '
"Press </a> last week that only 10% of needed humanitarian supplies have been reaching "
"Tigray in recent weeks, </p>\n <p>Five of the officials expelled work "
"with the U.N. humanitarian agency, another is with the U.N. human rights office and the "
"seventh is with UNICEF, the U.N. children’s agency.</p>",
}

remove_anchors(item)
self.assertNotIn("http", item.get("body_html"))

def test_story_with_embeds(self):
item = {
"body_html": '<p>one</p>\n<p>two</p>\n<!-- EMBED START Image {id: "editor_0"} -->\n<figure>\n '
'<img src="https://aa.bb.jpg" alt="" />\n <figcaption>EDITORIAL USE ONLY</figcaption>'
'</figure>\n<!-- EMBED END Image {id: "editor_0"} -->\n'
}

remove_anchors(item)
self.assertNotIn('http', item.get('body_html'))
self.assertIn("EMBED START", item.get("body_html"))

0 comments on commit d09c5a1

Please sign in to comment.