grab-files.lisp

(defun parse-type (row)
  (let* ((img (cl-ppcre:scan-to-strings "<img[^>]*>" row))
	 (alt (cl-ppcre:scan-to-strings "alt=.*" img)))
    (multiple-value-bind (_ v) (cl-ppcre:scan-to-strings "\"(.*)\"" alt)
      (declare (ignore _))
      (when v
	(aref v 0)))))

(defun parse-url (row)
  (multiple-value-bind (_ v) (cl-ppcre:scan-to-strings "href=\"([^\"]*)\"" row)
    (declare (ignore _))
    (when v
      (aref v 0))))

(defparameter files ())
(defparameter types ())
(defparameter *dir* "[DIR]")

(defun filep (type)
  (or (equalp type "[   ]")
      (equalp type "[TXT]")))

(defparameter base-url  "http://sources.vsta.org/forthos/src/")
(defparameter base-path "/Users/vyacheslavmikushev/Work/forthos/")

(defun populate-types (&optional (url base-url))
  (let ((page (drakma:http-request url)))
    (cl-ppcre:do-matches-as-strings (m "<tr>.*</tr>" page nil)
      (let ((type (parse-type m)))
	(when (equalp type *dir*)
	  (populate-types (concatenate 'string url (parse-url m))))
	(push type types)))))

(defun populate-files (&optional (url base-url) path)
  (let ((page (drakma:http-request url)))
    (cl-ppcre:do-matches-as-strings (m "<tr>.*</tr>" page nil)
      (let ((type (parse-type m)))
	(when (equalp type *dir*)
	  (let ((link (parse-url m)))
	    (populate-files (concatenate 'string url link) (cons link path))))
	(when (filep type)
	  (let ((link (parse-url m)))
	    (push (cons (cons link path) (concatenate 'string url link)) files)))))))

(defun save-file (props)
  (destructuring-bind (path . url) props
    (let* ((page (drakma:http-request url))
	   (page (if (stringp page)
		     page
		     (map 'string #'code-char page)))
	   (path (apply #'concatenate 'string base-path (reverse path)))
	   (path (parse-namestring path)))
      (ensure-directories-exist path)
      (with-open-file (str path
			   :direction :output
			   :if-exists :supersede
			   :if-does-not-exist :create
			   :external-format :utf-8)
	(write-string page str)))))

(defun save-files ()
  (dolist (file files)
    (format t "~a~%" file)
    (save-file file)))