go-boilerpipe example

main.go

package main

import (
	"encoding/csv"
	"net/http"
	"net/http/cookiejar"
	"os"

	"github.com/jlubawy/go-boilerpipe"
	"github.com/jlubawy/go-cli"
	"golang.org/x/net/publicsuffix"
)

func main() {
	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
	if err != nil {
		cli.Fatalf("error creating cookie jar: %v\n", err)
	}
	client := &http.Client{
		Jar: jar,
	}

	// Open input CSV file
	f, err := os.Open("comparision.csv")
	if err != nil {
		cli.Fatalf("error opening file: %v\n", err)
	}
	defer f.Close()
	r := csv.NewReader(f)

	// Open output CSV file
	of, err := os.Create("output.csv")
	if err != nil {
		cli.Fatalf("error opening output file: %v\n", err)
	}
	defer of.Close()
	w := csv.NewWriter(of)

	i := -1
	for i < 10 {
		rec, err := r.Read()
		if err != nil {
			cli.Fatalf("error reading csv: %v\n", err)
		}
		i += 1
		if i <= 0 {
			if err := w.Write(append(rec, "go-boilerpipe")); err != nil {
				cli.Fatalf("error writing record: %v\n", err)
			}
			continue
		}
		if len(rec) != 4 {
			cli.Fatalf("expected 4 entries: %v\n", rec)
		}

		u := rec[0]
		cli.Infof("downloading %s\n", u)

		resp, err := client.Get(u)
		if err != nil {
			cli.Fatalf("error getting url: %v\n", err)
		}
		if resp.StatusCode >= 400 {
			cli.Fatalf("received http error response: %d\n", resp.StatusCode)
		}

		doc, err := boilerpipe.ParseDocument(resp.Body)
		if err != nil {
			cli.Fatalf("error parsing response: %v\n", err)
		}
		boilerpipe.ArticlePipline.Process(doc)
		resp.Body.Close()

		if err := w.Write(append(rec, doc.Content())); err != nil {
			cli.Fatalf("error writing record: %v\n", err)
		}
		w.Flush()
	}
}

output.csv