do-me · November 18, 2023 08:09 · do-me · Nov 18, 2023
diff --git a/xml_html_parser.py b/xml_html_parser.py
 from lxml import etree
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pandas as pd
 from tqdm import tqdm
 import os
 tqdm.pandas()
 import lxml
 import cchardet
 import re

 def remove_consecutive_whitespaces(text):
    pattern = r'[^\S\r\n]*(\r\n|\n|\r)[^\S\r\n]*|([^\S\r\n]){2,}'
    return re.sub(pattern, lambda match: ' ' if match.group(2) else match.group(1), text)

 def write_parquet(file_path, data):
    if isinstance(data, str):
        # Assuming the string is a JSON representation of a DataFrame
        data = pd.DataFrame([data])
    elif not isinstance(data, (list, pd.DataFrame)):
        raise ValueError("Input data must be a list, DataFrame, or string")

    table = pa.Table.from_pandas(data)
    pq.write_table(table, file_path)

 def read_parquet(file_path):
    table = pq.read_table(file_path)
    return table.to_pandas()

 # shuffle 
 def extract_visible_text(xml_string):
    root = etree.fromstring(xml_string)
    visible_text = []

    for element in root.xpath('//text()'):
        # Check if the text is visible (non-empty and not just whitespace)
        if element.strip():
            visible_text.append(element.strip())

    result = "\n".join(visible_text)
    return result

 def extract_visible_text_from_html_file(x):
    with open(x, 'r', encoding='utf-8') as file:

        try:
            xml_data = file.read()

            # Use regex to extract each HTML content between <html> tags
            html_contents = re.findall(r'<html.*?</html>', xml_data, re.DOTALL)

            result = []
            for html_content in html_contents:
                result.append( extract_visible_text(html_content))
            
            result = "\n".join(result)

            result = re.sub(r'\n\s*\n', '\n', result).strip()
            result = remove_consecutive_whitespaces(result)
            return result

        except Exception as e:
            print(x,e)

 # Read the file list
 df = read_parquet("file_list.parquet")
 df = df.rename(columns={0:"filename"}) # input file has just one column named 0, needs to be renamed for saving later
 #df = df.iloc[:100]

 # Calculate the chunk size
 total_files = len(df)
 chunk_size = total_files // 20  # 5% of total files for each chunk

 processed_files = []

 # Process and save chunks
 for i in range(20):
    print(i)
    start_index = i * chunk_size
    end_index = (i + 1) * chunk_size if i < 19 else total_files

    chunk_df = df.iloc[start_index:end_index]

    # Process the chunk
    chunk_df["content"] = chunk_df["filename"].progress_apply(extract_visible_text_from_html_file)

    # Save the chunk to a Parquet file
    chunk_file_path = f"chunks_output/chunk_{i + 1}.parquet"
    chunk_df.to_parquet(chunk_file_path)

    # Add processed files to the list
    processed_files.extend(chunk_df["filename"].tolist())

 # Check for missing files
 original_files = df["filename"].tolist()
 missing_files = set(original_files) - set(processed_files)

 assert not missing_files, f"Missing files: {missing_files}"
 print("finished")

 # 17:40 m for  68492
	from lxml import etree
	import pyarrow as pa
	import pyarrow.parquet as pq
	import pandas as pd
	from tqdm import tqdm
	import os
	tqdm.pandas()
	import lxml
	import cchardet
	import re

	def remove_consecutive_whitespaces(text):
	pattern = r'[^\S\r\n](\r\n\|\n\|\r)[^\S\r\n]\|([^\S\r\n]){2,}'
	return re.sub(pattern, lambda match: ' ' if match.group(2) else match.group(1), text)

	def write_parquet(file_path, data):
	if isinstance(data, str):
	# Assuming the string is a JSON representation of a DataFrame
	data = pd.DataFrame([data])
	elif not isinstance(data, (list, pd.DataFrame)):
	raise ValueError("Input data must be a list, DataFrame, or string")

	table = pa.Table.from_pandas(data)
	pq.write_table(table, file_path)

	def read_parquet(file_path):
	table = pq.read_table(file_path)
	return table.to_pandas()

	# shuffle
	def extract_visible_text(xml_string):
	root = etree.fromstring(xml_string)
	visible_text = []

	for element in root.xpath('//text()'):
	# Check if the text is visible (non-empty and not just whitespace)
	if element.strip():
	visible_text.append(element.strip())

	result = "\n".join(visible_text)
	return result

	def extract_visible_text_from_html_file(x):
	with open(x, 'r', encoding='utf-8') as file:

	try:
	xml_data = file.read()

	# Use regex to extract each HTML content between <html> tags
	html_contents = re.findall(r'<html.*?</html>', xml_data, re.DOTALL)

	result = []
	for html_content in html_contents:
	result.append( extract_visible_text(html_content))

	result = "\n".join(result)

	result = re.sub(r'\n\s*\n', '\n', result).strip()
	result = remove_consecutive_whitespaces(result)
	return result

	except Exception as e:
	print(x,e)

	# Read the file list
	df = read_parquet("file_list.parquet")
	df = df.rename(columns={0:"filename"}) # input file has just one column named 0, needs to be renamed for saving later
	#df = df.iloc[:100]

	# Calculate the chunk size
	total_files = len(df)
	chunk_size = total_files // 20 # 5% of total files for each chunk

	processed_files = []

	# Process and save chunks
	for i in range(20):
	print(i)
	start_index = i * chunk_size
	end_index = (i + 1) * chunk_size if i < 19 else total_files

	chunk_df = df.iloc[start_index:end_index]

	# Process the chunk
	chunk_df["content"] = chunk_df["filename"].progress_apply(extract_visible_text_from_html_file)

	# Save the chunk to a Parquet file
	chunk_file_path = f"chunks_output/chunk_{i + 1}.parquet"
	chunk_df.to_parquet(chunk_file_path)

	# Add processed files to the list
	processed_files.extend(chunk_df["filename"].tolist())

	# Check for missing files
	original_files = df["filename"].tolist()
	missing_files = set(original_files) - set(processed_files)

	assert not missing_files, f"Missing files: {missing_files}"
	print("finished")

	# 17:40 m for 68492
No results found