image_down.go

/*go get github.com/mattn/go-sqlite3
/*go get github.com/PuerkitoBio/goquery
/*/
package main

import (
        "fmt"
        "io"
        "database/sql"
        "io/ioutil"
        "net/http"
        "os"
        "os/exec"
        "log"
        "path"
        "github.com/PuerkitoBio/goquery"
        "strings"
        _ "github.com/mattn/go-sqlite3"
)

func createTable(db *sql.DB) {
        createTableSQL := `CREATE TABLE img_urls (
        "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
        "imgurl" TEXT,
        "title" TEXT
        );` // SQL Statement for Create Table

        log.Println("Create table...")

        statement, err := db.Prepare(createTableSQL) // Prepare SQL Statement
        if err != nil {
                log.Fatal(err.Error())
        }
        statement.Exec() // Execute SQL Statements
        log.Println("table created.")
}

func main() {
        var url string = os.Args[1]

        fmt.Println(url)
        file, err := os.Create("images_url.db") // Create SQLite file
        if err != nil {
                log.Fatal(err.Error())
        }
        file.Close()
        log.Println("db created.")
        file = nil

        sqliteDatabase, _ := sql.Open("sqlite3", "./images_url.db") // Open the SQLite File
        defer sqliteDatabase.Close() // Defer Closing the database
        createTable(sqliteDatabase) // Create Database Tables


        //...sqlite insert variable
        id := 0
        imgURL := "image URL"
        var title string

        doc, err := goquery.NewDocument(url)
        if err != nil {
                panic("query failed.")
        }

        h1 := doc.Find("h1.entry-title")

        h1.Each(func(i int, s *goquery.Selection){
                str := s.Text()
                title = strings.TrimSpace(str)
                fmt.Println("title : ",strings.TrimSpace(str))
        })
        h1 = nil

        src := doc.Find("img")
                num := 0
                src.Each(func(i int, s *goquery.Selection){
                        str, _ := s.Attr("data-src")
                        num ++
                        id = num // to sql
                        imgURL = str //to sql
                        //fmt.Println(num,str)
                        //fmt.Println()
                        statement, _ := sqliteDatabase.Prepare("INSERT INTO img_urls (id,imgurl,title) VALUES (?,?,?) ;")
                        statement.Exec(id,imgURL,title)
                        log.Println(id,"...inserted!!")
        })
        src = nil
        doc = nil
//////////////////////////////////
//////////////////////////////////

        res := sqliteDatabase.QueryRow(`SELECT id FROM img_urls ORDER BY id DESC LIMIT 1;`)
        var lastid int
        switch err := res.Scan(&lastid); err {
        case sql.ErrNoRows:
                fmt.Println("No rows were returned!")
        case nil:
                fmt.Println("last_id : ",lastid)
        default:
                panic(err)
        }
        res = nil
        err = nil

        os.Mkdir("./" + title ,0777)

        for i := 1; i < lastid + 1; i++ {

                row := sqliteDatabase.QueryRow(`SELECT id,imgurl FROM img_urls WHERE id=?`,i)

                var  col_id int = 0
                var  col_imgurl string = ""

                err2 := row.Scan(&col_id,&col_imgurl);
                switch {
                        case err2 == sql.ErrNoRows:
                                log.Printf("nothing.")
                        case err2 != nil:
                                panic(err2)
                        default:
                                fmt.Printf("id: %d, imgurl: %s\n", col_id, col_imgurl)
                }
                row = nil
                err2 = nil

                response, err := http.Get(col_imgurl)

                if err != nil {
                        log.Println(err)
                }

                fmt.Println("title : ",title)

                fmt.Println("status:", response.Status)

                body, err := ioutil.ReadAll(response.Body)

                var readall = 0
                if err != nil {
                        log.Println(err)
                        readall = 1
                        //os.Exit(1)
                }
                err = nil

                filename := path.Base(col_imgurl)
                filename = fmt.Sprintf("%04d",col_id) + "_" + filename
                fmt.Println(filename)
                fmt.Println()


                file, err := os.OpenFile("./" + title + "/" + filename, os.O_CREATE|os.O_WRONLY, 0666)
                //file, err := os.Create("./" + title + "/" + filename)

                if err != nil {
                        log.Println(err)
                }
                err = nil

                defer file.Close()

                if readall == 1 {
                        _, err3 := io.Copy(file,response.Body)
                        if err3 != nil{
                                log.Println(err3)
                        }
                } else {
                        _, err3 := file.Write(body)
                        if err3 != nil{
                                log.Println(err3)
                        }
                }

        }

///////// zip
        cmd := exec.Command("zip","-4", title + ".zip","-r","./" + title)
        err = cmd.Run()

        if err != nil {
                log.Fatal(err)
        }
        cmd = nil
        cmd = exec.Command("rm","-r","./" + title)
        err = cmd.Run()

        if err != nil {
                log.Fatal(err)
        }
        cmd = nil
}