openrefine_external_call

client.py

import urllib.request
import json

# Define the URL of the Flask service
service_url = "https://5000-mongulucm-mtchounmouh-h28ndlzaxu8.ws-eu99.gitpod.io/extract_text"

# Define the URL to be scraped
scrape_url = "https://www.compta-online.com/oubli-de-facturation-facture-faite-posteriori-t72224#NULL"

# Combine these into the full request URL
request_url = service_url + "?url=" + urllib.parse.quote(scrape_url)

# Send the request and get the response
response = urllib.request.urlopen(request_url)

# Read the response data and decode it from bytes to a string
response_data = response.read().decode()

# Parse the JSON response
json_data = json.loads(response_data)

# Print the extracted text
print(json_data['text'])

webserver.py

from flask import Flask, request, jsonify
import requests
from bs4 import BeautifulSoup

app = Flask(__name__)

@app.route('/extract_text', methods=['GET'])
def extract_text():
    url = request.args.get('url', None)

    if url is None:
        return jsonify({'error': 'No URL provided.'}), 400

    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')

    # Find the input element with the name 'text'
    input_element = soup.find('input', {'name': 'text'})

    # Get its value (which is a URL)
    if input_element is not None:
        new_url = input_element.get('value')

        # Now fetch the new URL
        new_response = requests.get(new_url)
        new_soup = BeautifulSoup(new_response.text, 'html.parser')

        # Get all the text from the new page
        full_text = new_soup.get_text()

        # Find the start and end markers in the text
        start_marker = "ImprimerLienAlerter"
        end_marker = "Répondre"

        start = full_text.find(start_marker)
        end = full_text.find(end_marker)

        if start != -1 and end != -1:
            # Extract the text between the markers
            extracted_text = full_text[start+len(start_marker):end]
            return jsonify({'text': extracted_text})
        else:
            return jsonify({'error': 'Markers not found in the text.'}), 400
    else:
        return jsonify({'error': 'No input element with the name \'text\' found.'}), 400

if __name__ == '__main__':
    app.run(port=5000, debug=True)