PIXIV からダウンロードするやつ

pixiv-dl.scm

(define user-name "user-name") ;; PIXIV のユーザー名
(define user-password "password") ;; PIXIV のパスワード

(use rfc.http)
(use rfc.822)
(use rfc.cookie)
(use rfc.json)
(use rfc.uri)
(use rfc.md5)
(use gauche.parseopt)
(use gauche.generator)
(use gauche.collection)
(use srfi-13)
(use srfi-19)
(use html-unescape)
(use zip-archive)
(use www.cgi)

(define-class <pixiv> ()
  ((#:access-token :init-keyword :access_token :init-value #f
                   :accessor pixiv-access-token)
   (#:user-id :init-keyword :user-id :init-value #f
              :accessor pixiv-user-id)
   (#:refresh-token :init-keyword :refresh-token :init-value #f
                    :accessor pixiv-refresh-token)))

(define login-headers
  '(:App-OS "android"
    :App-OS-Version "5.0.156"
    :User-Agent "PixivAndroidApp/5.0.156 (Android 9; ONEPLUS A6013)"
    :accept-language "en_US"))

(define login-data
  '(("get_secure_url" "1")
    ("include_policy" "true")
    ("client_id" "MOBrBDS8blbauoSck0ZfDbtuzpyT")
    ("client_secret" "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj")))

(define hash-secret
  "28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c")

(define (result-check status body)
  (if (equal? status "200") body #f))

(define (pixiv-login obj id password)
  (define date (date->string (current-date) "~4"))
  (define date-hash
    (digest-hexify (md5-digest-string (string-append date hash-secret))))
  (receive (status header body)
      (apply http-post
             "oauth.secure.pixiv.net"
             "/auth/token"
             `(("grant_type" "password")
               ("username" ,id)
               ("password" ,password)
               ,@login-data)
             :secure #t
             :x-client-time date
             :x-client-hash date-hash
             login-headers)
    (and-let* ((result-text (result-check status body))
               (response (parse-json-string result-text))
               (access-token (~ response "response" "access_token"))
               (user-id (~ response "response" "user" "id"))
               (refresh-token (~ response "response" "refresh_token")))
      (set! (pixiv-access-token obj) access-token)
      (set! (pixiv-user-id obj) user-id)
      (set! (pixiv-refresh-token obj) refresh-token)
      #t)))

(define-method initialize ((obj <pixiv>) initargs)
  (let-keywords initargs ((name #f)
                          (password #f))
    (next-method)
    (when (and name password) (pixiv-login obj name password))))

(define make-pixiv
  (case-lambda
   ((id password) (make <pixiv> :name id :password password))
   (() (make <pixiv>))))

(define-method ref ((obj <list>) (key <string>) :optional (default #f))
  (assoc-ref obj key default))

(define-method ref ((obj <vector>) (key <string>))
  (let1 vlen (vector-length obj)
    (do ((i 0 (+ i 1))
         (result '()
                 (let1 r (ref (vector-ref obj i) key 'unfound)
                   (if (eqv? r 'unfound) result (cons r result)))))
        ((= i vlen) (list->vector result)))))

(define-method ref ((obj <symbol>) (key <string>))
  #f)

(define-method write-object ((obj <pixiv>) port)
  (format port "#<pixiv ~s ~s ~s>"
          (pixiv-access-token obj)
          (pixiv-user-id obj)
          (pixiv-refresh-token obj)))

(define request-headers
  '(:Referer "http://spapi.pixiv.net/"
             :User-Agent "PixivIOSApp/5.8.7"))

(define (pixiv-get obj url headers params)
  (receive (status header body)
      (apply http-get
             (uri-ref url 'host+port)
             (if (null? params)
                 (uri-ref url 'path)
                 (cons (uri-ref url 'path) params))
             :secure (equal? (uri-ref url 'scheme) "https")
             :Authorization #"Bearer ~(pixiv-access-token obj)"
             `(,@headers ,@request-headers))
    (result-check status body)))

(define (pixiv-next-url obj url)
  (let1 params (cgi-parse-parameters :query-string (uri-ref url 'query))
    (receive (status header body)
        (apply http-get
               (uri-ref url 'host+port)
               (if (null? params)
                   (uri-ref url 'path)
                   (cons (uri-ref url 'path) params))
               :secure (equal? (uri-ref url 'scheme) "https")
               :Authorization #"Bearer ~(pixiv-access-token obj)"
               `(,@login-headers ,@request-headers))
      (parse-json-string (result-check status body)))))

(define-syntax define-pixiv-api
  (syntax-rules ()
    ((_ name entry-point (must-arguments ...) (optional-arguments ...))
     (define (name session must-arguments ...
                   :key (optional-arguments #f) ...)
       (parse-json-string
        (pixiv-get session
                   (string-append "https://app-api.pixiv.net/" entry-point)
                   login-headers
                   `((,(symbol->string 'must-arguments) ,must-arguments)
                     ...
                     ,@(if optional-arguments
                           (list (list (symbol->string 'optional-arguments)
                                       optional-arguments))
                           '())
                     ...)))))))

(define-pixiv-api pixiv-illust-detail
  "v1/illust/detail" (illust_id) ())

(define-pixiv-api pixiv-ugoira-metadata
  "v1/ugoira/metadata" (illust_id) ())

(define-pixiv-api pixiv-user-illusts
  "v1/user/illusts" (user_id) (offset tag type))

(define-pixiv-api pixiv-user-novels
  "v1/user/novels" (user_id) (offset))

(define-pixiv-api pixiv-novel-text
  "v1/novel/text" (novel_id) ())

(define-pixiv-api pixiv-novel-detail
  "v2/novel/detail" (novel_id) ())

(define-pixiv-api pixiv-novel-series
  "v1/novel/series" (series_id) (offset))

(define (pixiv-illust-download obj url)
  (pixiv-get obj url '(:referer "https://app-api.pixiv.net/") '() ))

(define (url-extension url)
  (if-let1 m (#/\.(jpg|png|gif)$/ url)
    (m 1)
    #f))

(define (url-basename url)
  (if-let1 m (#/([^\/]+\.(?:jpg|png|gif))$/ url)
    (m 1)
    #f))

(define (sanitize title)
  (regexp-replace-all #/[\\\/'()"?<>|:;*~\r\n]/ title "_"))

(define (make-filename author-id id title ext series-id)
  (let1 series-id (if series-id #"-~|series-id|" "")
    #"PIXIV ~|author-id|~|series-id|-~|id| ~(sanitize title).~|ext|"))

(define (pixiv-single-illust-save obj info)
  (let* ((title (~ info "title" ))
         (illust-id (~ info "id"))
         (author-id (~ info "user" "id"))
         (url (~ info "meta_single_page" "original_image_url"))
         (data (pixiv-illust-download obj url))
         (series-id (~ info "series" "id")))
    (call-with-output-file
        (make-filename author-id illust-id title (url-extension url) series-id)
      (pa$ display data))))

(define (pixiv-ugoira-save obj info)
  (let* ((title (~ info "title" ))
         (illust-id (~ info "id"))
         (author-id (~ info "user" "id"))
         (url (~ (pixiv-ugoira-metadata obj illust-id)
                 "ugoira_metadata" "zip_urls" "medium"))
         (data (pixiv-illust-download obj url))
         (series-id (~ info "series" "id")))
    (call-with-output-file (make-filename author-id illust-id title "zip" series-id)
      (pa$ display data))))

(define (pixiv-illusts-packing obj info)
  (let ((title (~ info "title" ))
        (illust-id (~ info "id"))
        (author-id (~ info "user" "id"))
        (series-id (~ info "series" "id")))
    (call-with-output-zip-archive
     (make-filename author-id illust-id title "zip" series-id)
     (lambda(zip)
       (vector-for-each
        (lambda(x)
          (let1 series-label (if series-id #"-~|series-id|" "")
            (zip-add-entry zip
                           #"PIXIV ~|author-id|~|series-label|-~(url-basename x)"
                           (pixiv-illust-download obj x))))
        (~ info "meta_pages" "image_urls" "original"))))))

(define (pixiv-illust-save obj info)
  (and-let* ((type (~ info "type"))
             (title (~ info "title"))
             (illust-id (~ info "id")))
    (cond
     ((equal? type "ugoira")
      (pixiv-ugoira-save obj info))
     ((or (equal? type "illust") (equal? type "manga"))
      (if (equal? (~ info "page_count") 1)
          (pixiv-single-illust-save obj info)
          (pixiv-illusts-packing obj info))))))

(define (usage cmd)
  (print "usage: " cmd " [options] [args]")
  (print "\
Options:
  --listfile file   reading list from file.
")
  (exit))

(define (parameter-ref key obj :optional (default #f))
  (cgi-get-parameter key obj :default default))

(define (pixiv-illusts self path)
  (cond
   ((#/^\/artworks\/(\d+)$/ path)
    (and-let*
        ((illust-id ((#/^\/artworks\/(\d+)$/ path) 1))
         (detail (pixiv-illust-detail self illust-id)))
      (pixiv-illust-save self (~ detail "illust"))))))

(define (pixiv-illusts-crawl self query)
  (cond
   ((parameter-ref "id" query)
    (let ((type (parameter-ref "type" query))
          (tag (parameter-ref "tag" query)))
      (let next
          ((is (pixiv-user-illusts self (parameter-ref "id" query))))
        (vector-for-each
         (lambda(x) (if (and (or (not tag)
                                 (find (pa$ equal? tag) (~ x "tags" "name")))
                             (or (not type)
                                 (equal? (~ x "type") type)))
                        (pixiv-illust-save self x)))
         (~ is "illusts"))
        (and-let* ((n (~ is "next_url"))
                   ((not (eqv? n 'null))))
          (next (pixiv-next-url self n))))))))

(define (pixiv-novel-save obj id)
  (let* ((detail (pixiv-novel-detail obj id))
         (text (pixiv-novel-text obj id))
         (title (~ detail "novel" "title"))
         (author-id (~ detail "novel" "user" "id"))
         (series-id (~ detail "novel" "series" "id")))
    (call-with-output-file (make-filename author-id id title "txt" series-id)
      (lambda(p)
        (display (~ detail "novel" "title") p)
        (newline p)
        (display (~ text "novel_text") p)))))

(debug-print-width #f)
(define (pixiv-novel-series-crawl obj id)
  (let next ((series (pixiv-novel-series obj id)))
    (vector-for-each
     (lambda(x)
       (let* ((id (~ x "id"))
              (text (pixiv-novel-text obj id))
              (author-id (~ x "user" "id"))
              (title (~ x "title"))
              (series-id (~ x "series" "id")))
         (call-with-output-file (make-filename author-id id title "txt" series-id)
           (lambda(p)
             (display (~ x "title") p)
             (newline p)
             (display (~ text "novel_text") p)))))
     (~ series "novels"))
    (and-let* ((n (~ series "next_url"))
               ((not (eqv? n 'null))))
      (next (pixiv-next-url obj n))
      )))

(define (pixiv-user-novels-crawl obj query)
  (let1 tag (parameter-ref "tag" query)
    (let next ((series (pixiv-user-novels obj (parameter-ref "id" query))))
      (vector-for-each
       (lambda(x)
         (if (or (not tag)
                 (find (pa$ equal? tag) (~ x "tags" "name")))
             (let1 text (pixiv-novel-text obj (~ x "id"))
               (call-with-output-file
                   (make-filename (~ x "user" "id") (~ x "id") (~ x "title") "txt" (~ x "series" "id"))
                 (lambda(p)
                   (display (~ x "title") p)
                   (newline p)
                   (display (~ text "novel_text") p))))))
       (~ series "novels"))
      (and-let* ((n (~ series "next_url"))
                 ((not (eqv? n 'null))))
        (next (pixiv-next-url obj n))))))

(define (get pixiv uri)
  (receive (scheme user-info hostname port path query-a fragment)
      (uri-parse uri)
    (and-let* (((or (equal? scheme "https") (equal? scheme "http")))
               ((not user-info))
               ((equal? hostname "www.pixiv.net"))
               ((or (equal? port 443) (equal? port #f)))
               (query-b (if query-a query-a ""))
               (query (cgi-parse-parameters :query-string query-b)))
      (cond ((string-prefix? "/artworks" path)
             (pixiv-illusts pixiv path))
            ((equal? path "/member_illust.php")
             (pixiv-illusts-crawl pixiv query))
            ((equal? path "/novel/member.php")
             (pixiv-user-novels-crawl pixiv query))
            ((equal? path "/novel/show.php")
             (pixiv-novel-save pixiv (parameter-ref "id" query)))
            ((equal? path "/series.php")
             (pixiv-novel-series-crawl pixiv (parameter-ref "id" query)))
            (else (error "invalid id : " uri))))))

(define file->list
  (cut call-with-input-file <> (pa$ port->list read-line)))

(define (main args)
  (let-args (cdr args)
      ((listfile  "l|listfile=s"  #f)
       (help  "h|help" => (cut usage (car args)))
       . targets)
    (let1 targets (if listfile (file->list listfile) targets)
      (if (null? targets)
          (usage (car args))
          (let1 pixiv (make-pixiv user-name user-password)
            (for-each
             (lambda(x)
               ;;(guard (e (else (error #"error when download ~|x|")))
               (get pixiv x))
             ;;)
             targets))))))