dtcc-2017.go

/* download ppt pdf from dtcc 2017
 * Author: Mindon <mindon@gmail.com>  http://mindon.github.io
 */
package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

func main() {

	resp, err := http.Get("http://dtcc.it168.com/DTCC2017PPT/")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		panic(err)
	}

	rxp := regexp.MustCompile(`<li><a href="([^"]+)"[^>]*>• ([^<]+)<\/a>`)

	links := rxp.FindAllSubmatch(body, -1)

	var wg sync.WaitGroup
	fmt.Printf("Topics = %d\n", len(links))
	for _, link := range links {
		wg.Add(1)
		go func(topicUrl string, topic string) {
			defer wg.Done()

			resp, err := http.Get(topicUrl)
			if err != nil {
				fmt.Println("Topic failed", err, topic)
				return
			}
			defer resp.Body.Close()
			body, err := ioutil.ReadAll(resp.Body)

			if err != nil {
				fmt.Println("Topic failed", err, topic)
				return
			}

			fldir := filepath.Join("dtcc", topic)
			if _, err := os.Stat(fldir); os.IsNotExist(err) {
				os.MkdirAll(fldir, os.ModePerm)
			}

			docs := rxp.FindAllSubmatch(body, -1)
			fmt.Printf("Subjects on %s = %d\n", topic, len(docs))

			var wgDoc sync.WaitGroup
			for _, doc := range docs {
				docUrl := string(doc[1])
				subject := strings.Replace(string(doc[2]), ":", "-", -1)
				i := strings.LastIndex(docUrl, ".")
				docExt := ".pdf"
				if i > 0 {
					docExt = docUrl[i:]
				}
				flpath := filepath.Join(fldir, subject+docExt)
				if _, err := os.Stat(flpath); !os.IsNotExist(err) {
					continue
				}
				wgDoc.Add(1)

				go func(docUrl string, subject string) {
					err := download(docUrl, flpath)
					if err != nil {
						fmt.Println("[FAILED] ", err, topic, subject)
					} else {
						fmt.Println(topic, subject)
					}
					wgDoc.Done()
				}(docUrl, subject)
			}
			wgDoc.Wait()
		}(string(link[1]), string(link[2]))
	}
	wg.Wait()
}

func download(docUrl string, flpath string) error {
	resp, err := http.Get(docUrl)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	f, err := os.Create(flpath)
	if err != nil {
		return err
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	w.Write(body)
	w.Flush()
	return nil
}