Created
June 10, 2023 16:19
-
-
Save billmetangmo/b606a5ed328a5634c7090b3cf02fb2c2 to your computer and use it in GitHub Desktop.
scrape email from websites
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # katana -list urls.txt -fs dn -j -o pages.json -silent | |
| import json | |
| import re | |
| import csv | |
| from concurrent.futures import ThreadPoolExecutor | |
| def find_email_in_json_line(json_line): | |
| # Parse the JSON line | |
| data = json.loads(json_line) | |
| # Extract the endpoint | |
| endpoint = data['request']['endpoint'] | |
| # Check if 'body' exists in 'response' | |
| if 'body' in data['response']: | |
| # Extract the body | |
| body = data['response']['body'] | |
| # Use a regular expression to find emails in the body | |
| email_regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b' | |
| emails = re.findall(email_regex, body) | |
| # If any emails are found, return them along with the endpoint | |
| if emails: | |
| return [(endpoint, email) for email in emails] | |
| return None | |
| def process_file(file_name, output_csv): | |
| # Open the output CSV file | |
| with open(output_csv, 'w', newline='') as csvfile: | |
| writer = csv.writer(csvfile) | |
| writer.writerow(['endpoint', 'email']) | |
| # Use a thread pool to process the lines in parallel | |
| with ThreadPoolExecutor() as executor: | |
| with open(file_name, 'r') as file: | |
| futures = {executor.submit(find_email_in_json_line, line) for line in file} | |
| for future in futures: | |
| result = future.result() | |
| if result: | |
| for endpoint, email in result: | |
| writer.writerow([endpoint, email]) | |
| process_file('pages.json', 'output.csv') |
Author
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
sample url.txt file