Skip to content

Instantly share code, notes, and snippets.

@vfig

vfig/download_guide_pdf.go

Forked from RoCry/download_guide_pdf.go
Created January 13, 2020 11:45
Show Gist options
  • Save vfig/2d72f0609ad68499f0b6ae4b337b9178 to your computer and use it in GitHub Desktop.
Save vfig/2d72f0609ad68499f0b6ae4b337b9178 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
download_guide_pdf.go
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"os/exec"
"strconv"
"strings"
"github.com/jmoiron/jsonq"
)
func main() {
resp, err := http.Get("https://developer.apple.com/library/ios/navigation/library.json")
PanicIfError(err)
defer resp.Body.Close()
data := map[string]interface{}{}
dec := json.NewDecoder(resp.Body)
dec.Decode(&data)
jq := jsonq.NewQuery(data)
// get guides' key
contents, err := jq.ArrayOfObjects("topics", "0", "contents")
PanicIfError(err)
var guideKey int
for _, v := range contents {
name := v["name"]
if name == "Guides" {
keyStr := v["key"].(string)
guideKey, err = strconv.Atoi(keyStr)
PanicIfError(err)
break
}
}
// assmenble url
basePath := "https://developer.apple.com/library/ios"
documents, err := jq.ArrayOfArrays("documents")
var guideDocPaths []string
for _, v := range documents {
key := v[2].(float64)
if int(key) == guideKey {
relativePath := v[9].(string)
if relativePath[0:3] != "../" {
fmt.Println("maybe some error at:", relativePath)
continue
}
relativePath = relativePath[2:len(relativePath)]
path := basePath + relativePath
guideDocPaths = append(guideDocPaths, path)
// fmt.Println(path)
}
}
// fmt.Println("first guide path:", guideDocPaths[0])
fmt.Println("library json loaded")
downloadPdf(guideDocPaths)
}
func downloadPdf(guideDocPaths []string) {
// download pdf
// https://developer.apple.com/library/ios/documentation/Performance/Conceptual/ManagingMemory/book.json
// https://developer.apple.com/library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
downloadedFileNames := make(chan string, len(guideDocPaths))
workers := make(chan bool, 10)
for _, v := range guideDocPaths {
workers <- true
go func(guideDocPath string) {
fmt.Println("begin parse:", guideDocPath)
u, err := url.Parse(guideDocPath)
PanicIfError(err)
oldPath := u.Path
u.Fragment = ""
// get book.json
// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/Introduction/Introduction.html
// =>>>
// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
oldComps := strings.Split(oldPath, "/")
lastNeededCompIndex := len(oldComps) - 1
TryGetBookJSON:
newComps := oldComps[0:lastNeededCompIndex]
u.Path = strings.Join(newComps, "/") + "/book.json"
bookFullPath := u.String()
// get Pdf file name and download link
resp, err := http.Get(bookFullPath)
PanicIfError(err)
defer resp.Body.Close()
if resp.StatusCode != 200 {
lastNeededCompIndex--
if lastNeededCompIndex >= 0 {
goto TryGetBookJSON
} else {
fmt.Println("ERROR:", err, "oldPath:", oldPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
}
data := map[string]interface{}{}
dec := json.NewDecoder(resp.Body)
dec.Decode(&data)
jq := jsonq.NewQuery(data)
// fmt.Println("begin find pdf link for:", bookFullPath)
pdfName, err := jq.String("PDF", "href")
if err != nil {
fmt.Println("find pdf href err:", err, " bookFullPath:", bookFullPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
// fmt.Println("pdfName:::", pdfName)
pdfPath := strings.Join(newComps, "/") + "/" + pdfName
u.Path = pdfPath
pdfFullPath := u.String()
// fmt.Println("pdfFullPath:::", pdfFullPath)
// download it
fmt.Println("begin download:", pdfFullPath)
cmd := exec.Command("curl", pdfFullPath, "-o", pdfName)
// var out bytes.Buffer
// cmd.Stdout = &out
// cmd.Stderr = &out
err = cmd.Run()
if err != nil {
fmt.Println("download pdf err:", err, " pdfFullPath:", pdfFullPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
downloadedFileNames <- pdfName
<-workers
return
}(v)
}
for i := 0; i < len(guideDocPaths); i++ {
<-downloadedFileNames
}
fmt.Println("----------DONE----------")
}
func releaseWorkerAndFileNames(downloadedFileNames chan string, workers chan bool) {
<-workers
downloadedFileNames <- "DOWNLOAD FILE ERROR!!"
}
func PanicIfError(err error) {
if err != nil {
log.Fatal(err)
}
}
Raw
download_prerelease_sample_code.go
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"os/exec"
"strconv"
"strings"
"github.com/jmoiron/jsonq"
)
func main() {
resp, err := http.Get("https://developer.apple.com/library/prerelease/ios/navigation/library.json")
PanicIfError(err)
defer resp.Body.Close()
data := map[string]interface{}{}
dec := json.NewDecoder(resp.Body)
dec.Decode(&data)
jq := jsonq.NewQuery(data)
// get guides' key
contents, err := jq.ArrayOfObjects("topics", "0", "contents")
PanicIfError(err)
var guideKey int
for _, v := range contents {
name := v["name"]
if name == "Sample Code" {
keyStr := v["key"].(string)
guideKey, err = strconv.Atoi(keyStr)
PanicIfError(err)
break
}
}
// fmt.Println("key = ", guideKey)
// assmenble url
basePath := "https://developer.apple.com/library/prerelease/ios"
documents, err := jq.ArrayOfArrays("documents")
var guideDocPaths []string
for _, v := range documents {
key := v[2].(float64)
if int(key) == guideKey {
relativePath := v[9].(string)
if relativePath[0:3] != "../" {
fmt.Println("maybe some error at:", relativePath)
continue
}
relativePath = relativePath[2:len(relativePath)]
path := basePath + relativePath
guideDocPaths = append(guideDocPaths, path)
// fmt.Println(path)
}
}
// fmt.Println("first guide path:", guideDocPaths[0])
fmt.Println("library json loaded")
downloadPdf(guideDocPaths)
}
func downloadPdf(guideDocPaths []string) {
// download pdf
// https://developer.apple.com/library/ios/documentation/Performance/Conceptual/ManagingMemory/book.json
// https://developer.apple.com/library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
downloadedFileNames := make(chan string, len(guideDocPaths))
workers := make(chan bool, 10)
for _, v := range guideDocPaths {
workers <- true
go func(guideDocPath string) {
// fmt.Println("begin parse:", guideDocPath)
u, err := url.Parse(guideDocPath)
PanicIfError(err)
oldPath := u.Path
u.Fragment = ""
// get book.json
// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/Introduction/Introduction.html
// =>>>
// /library/ios/documentation/General/Conceptual/CocoaTouch64BitGuide/book.json
oldComps := strings.Split(oldPath, "/")
lastNeededCompIndex := len(oldComps) - 1
TryGetBookJSON:
newComps := oldComps[0:lastNeededCompIndex]
u.Path = strings.Join(newComps, "/") + "/book.json"
bookFullPath := u.String()
// get Pdf file name and download link
fmt.Println("before get: ", bookFullPath)
resp, err := http.Get(bookFullPath)
PanicIfError(err)
defer resp.Body.Close()
if resp.StatusCode != 200 {
lastNeededCompIndex--
if lastNeededCompIndex >= 0 {
goto TryGetBookJSON
} else {
fmt.Println("ERROR:", err, "oldPath:", oldPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
}
data := map[string]interface{}{}
dec := json.NewDecoder(resp.Body)
dec.Decode(&data)
jq := jsonq.NewQuery(data)
// fmt.Println("begin find pdf link for:", bookFullPath)
pdfName, err := jq.String("sampleCode")
if err != nil {
fmt.Println("find pdf href err:", err, " bookFullPath:", bookFullPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
// fmt.Println("pdfName:::", pdfName)
pdfPath := strings.Join(newComps, "/") + "/" + pdfName
u.Path = pdfPath
pdfFullPath := u.String()
// fmt.Println("pdfFullPath:::", pdfFullPath)
// download it
fmt.Println("begin download:", pdfFullPath)
cmd := exec.Command("curl", pdfFullPath, "-o", pdfName)
// var out bytes.Buffer
// cmd.Stdout = &out
// cmd.Stderr = &out
err = cmd.Run()
if err != nil {
fmt.Println("download pdf err:", err, " pdfFullPath:", pdfFullPath)
releaseWorkerAndFileNames(downloadedFileNames, workers)
return
}
downloadedFileNames <- pdfName
<-workers
return
}(v)
}
for i := 0; i < len(guideDocPaths); i++ {
<-downloadedFileNames
}
fmt.Println("----------DONE----------")
}
func releaseWorkerAndFileNames(downloadedFileNames chan string, workers chan bool) {
<-workers
downloadedFileNames <- "DOWNLOAD FILE ERROR!!"
}
func PanicIfError(err error) {
if err != nil {
log.Fatal(err)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment