111 lines
2.8 KiB
Python
111 lines
2.8 KiB
Python
import re
|
|
|
|
import bs4
|
|
|
|
HTML_TAGS_MOSTLY_CONTENTLESS: set[str] = {
|
|
'style',
|
|
'svg',
|
|
'link',
|
|
'br',
|
|
'math',
|
|
'canvas',
|
|
}
|
|
|
|
HTML_TAGS_WITH_LITTLE_CONTENT: set[str] = {
|
|
'head',
|
|
'script',
|
|
'meta',
|
|
} | HTML_TAGS_MOSTLY_CONTENTLESS
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
text = text.replace('\t', ' ')
|
|
text = text.replace('\r', '')
|
|
text = re.sub(r'\s*\n\s*\n\s*', '\n\n', text)
|
|
text = re.sub(r' +', ' ', text)
|
|
text = re.sub(r'^\s+', '', text)
|
|
text = re.sub(r'\s+$', '', text)
|
|
return text.encode('utf-8')
|
|
|
|
|
|
def normalize_soup_bs4(soup: bs4.BeautifulSoup) -> bytes:
|
|
for comment in soup(text=lambda text: isinstance(text, bs4.Comment)):
|
|
comment.extract()
|
|
del comment
|
|
for element_name in HTML_TAGS_WITH_LITTLE_CONTENT:
|
|
for script_elements in soup(element_name):
|
|
script_elements.decompose()
|
|
del element_name
|
|
soup.smooth()
|
|
return soup
|
|
|
|
|
|
def normalize_soup_lxml(soup) -> bytes:
|
|
for element_name in HTML_TAGS_WITH_LITTLE_CONTENT:
|
|
for script_elements in soup.cssselect(element_name):
|
|
script_elements.drop_tree()
|
|
del script_elements
|
|
del element_name
|
|
return soup
|
|
|
|
|
|
def normalize_soup(soup) -> bytes:
|
|
text = None
|
|
if isinstance(soup, bs4.BeautifulSoup):
|
|
text = normalize_soup_bs4(soup).get_text()
|
|
else:
|
|
text = normalize_soup_lxml(soup).text_content()
|
|
return normalize_text(text)
|
|
|
|
|
|
def data_attributes_of_element(e):
|
|
for attr_key in list(e.attrs.keys()):
|
|
if attr_key.startswith('data-'):
|
|
yield attr_key
|
|
|
|
|
|
def has_data_attribute(e) -> bool:
|
|
for attr_key in data_attributes_of_element(e):
|
|
return True
|
|
return False
|
|
|
|
|
|
def normalize_soup_slightly(
|
|
soup, classes=True, scripts=True, comments=True, data_attributes=True,
|
|
):
|
|
"""Perform soup normalization."""
|
|
# Little if any content
|
|
for tag in HTML_TAGS_MOSTLY_CONTENTLESS:
|
|
for e in soup.select(tag):
|
|
e.decompose()
|
|
|
|
if classes:
|
|
for e in soup.find_all(class_=True):
|
|
del e['class']
|
|
for e in soup.find_all('script', src=True):
|
|
e.decompose()
|
|
for e in soup.find_all(style=True):
|
|
del e['style']
|
|
for e in soup.select('a'):
|
|
del e['height'], e['target'], e['rel'], e['onclick']
|
|
|
|
for e in soup.select('a[href=""]'):
|
|
del e['href']
|
|
|
|
if scripts:
|
|
for e in soup.find_all('script'):
|
|
e.decompose()
|
|
|
|
if comments:
|
|
for c in soup.find_all(string=lambda text: isinstance(text, bs4.Comment)):
|
|
c.extract()
|
|
|
|
if data_attributes:
|
|
for e in soup.find_all(has_data_attribute):
|
|
for attr_key in data_attributes_of_element(e):
|
|
del e[attr_key], attr_key
|
|
del e
|
|
|
|
soup.smooth()
|
|
return soup
|