Inecobank scraping

1password.lisp

(in-package :yobabank)

(defparameter *1pw-secure-note-name* "Yobabank Data")
(defparameter *1pw-vault* "Personal")

(defun 1pw-get-field (object name)
  (when (typep object 'st-json:jso)
    (st-json:getjso name object)))

(defun 1pw-get-id (object)
  (1pw-get-field object "id"))

(defun 1pw-value-by-id (object id)
  (or (1pw-get-field (find id object :key #'1pw-get-id :test #'equal) "value")
      (error "can't get value of 1password field ~s" id)))

(defun 1pw-get (name fields &optional (vault *1pw-vault*))
  (st-json:read-json-from-string
   (uiop:run-program (list "op" "item" "get" name "--fields"
                           (format nil "~{label=~a~^,~}" fields)
                           "--format" "json"
                           "--vault" vault)
                     :output :string
                     :error-output t)))

(defun 1pw-get-login (name &optional (vault *1pw-vault*))
  (let ((object (1pw-get name '("username" "password") vault)))
    (values (1pw-value-by-id object "username")
            (1pw-value-by-id object "password"))))

(defun 1pw-delete-item (name &optional (vault *1pw-vault*))
  (ignore-errors (uiop:run-program (list "op" "item" "delete" name "--vault" vault))))

(defun 1pw-store-secure-note (data &optional (name *1pw-secure-note-name*) (vault *1pw-vault*))
  (let ((text (with-standard-io-syntax
                (with-output-to-string (out)
                  (print data out)))))
    (uiop:with-temporary-file (:stream out :pathname out-path :direction :output)
      (st-json:write-json
       (st-json:jso "fields"
                    (list (st-json:jso "id" "notesPlain"
                                       "type" "STRING"
                                       "purpose" "NOTES"
                                       "label" "notesPlain"
                                       "value" text)))
       out)
      (finish-output out)
      (1pw-delete-item name vault)
      (uiop:run-program (list "op" "item" "create"
                              "--category" "securenote"
                              "--title" name
                              "--vault" vault
                              "--template" (namestring out-path))
                        :output nil
                        :error-output t))))

(defun 1pw-read-secure-note (&optional (name *1pw-secure-note-name*) (vault *1pw-vault*))
  (when-let ((object (ignore-errors (1pw-get name '("notesPlain") vault))))
    (with-standard-io-syntax
      (read-from-string (1pw-get-field object "value")))))

(defvar *secret*)

(defun reset-secret ()
  (1pw-delete-item *1pw-secure-note-name*)
  (makunbound '*secret*))

(defun ensure-secret-loaded ()
  (unless (boundp '*secret*)
    (setf *secret* (1pw-read-secure-note))))

(defun store-secrets ()
  (if (boundp '*secret*)
      (1pw-store-secure-note *secret*)
      (warn "Secret not loaded")))

(defun secret-get (name &optional default)
  (ensure-secret-loaded)
  (getf *secret* name default))

(defun (setf secret-get) (value name &optional default)
  (declare (ignore default))
  (ensure-secret-loaded)
  (setf (getf *secret* name) value))

ineco-web.lisp

(in-package :yobabank)

;; <input class="dxeTextBoxSys" id="txtToken" type="text" name="txtToken" onchange="aspxEValueChanged('txtToken')" onkeydown="aspxEKeyDown('txtToken', event)" onkeypress="aspxEKeyPress('txtToken', event)" autocomplete="off">
;; <input title="Get security code by sms" class="dxb-hb" value="Get security code by SMS" type="button" name="btnSMS">
;; succcessful login:
;; <table class="balancetbl" cellspacing="0" cellpadding="0" border="0">

(defun save-screenshot ()
  (let ((b64 (screenshot)))
    (with-open-file (out "/Users/ivan4th/rmme/a.png" :direction :output
			                             :if-does-not-exist :create
			                             :if-exists :supersede
			                             :element-type '(unsigned-byte 8))
      (write-sequence (base64:base64-string-to-usb8-array b64) out)
      (values))))

(defun execute-cdp-command (command &rest args)
  (webdriver::http-post-value
   (webdriver::session-path webdriver::*session* "/goog/cdp/execute")
   :cmd command
   :params (plist-hash-table args :test #'equal)))

(defun grab-webdriver-cookies (key)
  (setf (secret-get key)
        (iter (for cookie in (webdriver:cookie))
              (collect (remove-from-plist (alist-plist cookie) :same-site)))))

;; https://stackoverflow.com/questions/63220248/how-to-preload-cookies-before-first-request-with-python3-selenium-chrome-webdri
;; https://github.com/falqondev/selenium/blob/1a5cfaacd059afd42f499ac8574e661999151e9d/remote.go#L1231-L1269
;; https://github.com/screenshotbot/screenshotbot-oss/blob/4cc115b7194fe512c736b86dab7114f20ff7f40d/src/screenshotbot/webdriver/screenshot.lisp#L74

(defun set-webdriver-cookies (key)
  (execute-cdp-command "Network.enable")
  (iter (for cookie in (secret-get key))
        (destructuring-bind (&key domain expiry http-only name path secure value &allow-other-keys)
            cookie
          (apply #'execute-cdp-command
                 "Network.setCookie"
                 "domain" domain
                 "http-only" http-only
                 "name" name
                 "path" path
                 "secure" (or secure :false)
                 "value" value
                 (when expiry
                   ;; sic!
                   (list "expires" expiry)))))
  (execute-cdp-command "Network.disable"))

(defun wait-and-dismiss-alert ()
  (iter (repeat 10)
        (while (null (ignore-errors (webdriver:alert-text))))
        (sleep 1)
        (finally
            (when (ignore-errors (webdriver:alert-text))
              (webdriver:dismiss-alert)))))

(defun ineco-web-login (key credentials &optional (interactive-session-p t))
  (dbg "Inecobank Web login: ~s ~s" key credentials)
  (multiple-value-bind (login password)
      (1pw-get-login credentials)
    (flet ((doit ()
             (dbg "Initializing cookies")
             (webdriver:delete-all-cookies)
             (set-webdriver-cookies key)
             (dbg "Loading Inecobank page")
             (setf (webdriver:url) "https://online.inecobank.am/")
             (wait-for "#txtUserName,.balancetbl")
             (unless (find-elem ".balancetbl")
               (dbg "Logging in")
               (let ((elem (find-elem "#txtUserName")))
                 (webdriver:element-clear elem)
                 (webdriver:element-send-keys elem login))
               (let ((elem (find-elem "#txtPassword")))
                 (webdriver:element-clear elem)
                 (webdriver:element-send-keys elem password))
               (save-screenshot)
               (click "#btnSubmit")
               (wait-for "[name=btnSMS],.balancetbl")
               (let ((ts (load-last-sms-code)))
                 (unwind-protect
                      (when-let ((elem (find-elem "#txtToken")))
                        (dbg "Performing 2FA")
                        (click "#btnSMS_CD")
                        (wait-and-dismiss-alert)
                        (webdriver:element-send-keys elem (wait-for-sms-code :start-ts ts))
                        (click "#btnSubmit")
                        (wait-for ".balancetbl"))
                   (save-screenshot))))
             (dbg "Loading cookies")
             (grab-webdriver-cookies key)
             (dbg "Storing secrets")
             (store-secrets)))
      (let ((caps (make-capabilities
                   :always-match '((:browser-name . "chrome"))
                   :first-match (list
                                 '((:platform-name . "macos"))
                                 '((:platform-name . "linux"))))))
        (cond (interactive-session-p
               (webdriver:start-interactive-session caps)
               (doit))
              (t
               (with-session caps
                 (doit))))))))

(defun cookie-jar-from-secret (key)
  (make-instance
   'drakma:cookie-jar
   :cookies
   (iter (for cookie in (secret-get key))
         (destructuring-bind (&key domain expiry http-only name path secure value &allow-other-keys) cookie
           (collect
               (make-instance 'drakma:cookie
                              :name name
                              :value value
                              :path path
                              :expires expiry
                              :domain domain
                              :securep secure
                              :http-only-p http-only))))))

#++
(defun parse-cookie-bool (str)
  (cond ((string-equal "TRUE" str) t)
        ((string-equal "FALSE" str) nil)
        (t
         (warn "invalid cookie bool: ~s" str)
         nil)))

#++
(defun parse-cookies-txt (path)
  (make-instance
   'drakma:cookie-jar
   :cookies
   (with-open-file (in path)
     (iter (for line = (read-line in nil nil))
           (while line)
           (let ((trimmed (trim line)))
             (unless (or (emptyp trimmed)
                         (starts-with #\# trimmed))
               (let ((parts (split-sequence:split-sequence #\tab trimmed)))
                 (if (length= 7 parts)
                     (destructuring-bind (domain flag path secure expiration name value)
                         parts
                       (declare (ignore flag))
                       (collect (make-instance 'drakma:cookie
                                               :name name
                                               :value value
                                               :path path
                                               :expires (let ((v (cond ((parse-integer expiration))
                                                                       (t
                                                                        (warn "Failed to parse expiration time: ~s"
                                                                              expiration)
                                                                        0))))
                                                          (if (plusp v)
                                                              (unix-timestamp->universal-time v)
                                                              nil))
                                               :domain domain
                                               :securep (parse-cookie-bool secure)
                                               ;; FIXME
                                               :http-only-p nil)))
                     (warn "invalid cookie line:~%~a" line)))))))))

(defparameter *ineco-web-user-agent* "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0")

#++
(defparameter *ineco-web-cookie-file* #p"/tmp/cookies-online-inecobank-am.txt")
(defparameter *ineco-web-account-prefix* "20528")
(defparameter *ineco-web-statement-dir* #p "/Users/ivan4th/work/ledger/statements")

(defvar *loaded-statements* (make-hash-table :test #'equal))

(defun load-statement (account cookie-key)
  (when (numberp account)
    (setf account (princ-to-string account)))
  (when (starts-with-subseq *ineco-web-account-prefix* account)
    (setf account (subseq account (length *ineco-web-account-prefix*))))
  ;; TBD: handle errors (html page)
  (babel:octets-to-string
   (drakma:http-request "https://online.inecobank.am/AccountStatement/Export"
                        :method :post
                        :user-agent *ineco-web-user-agent*
                        :cookie-jar (cookie-jar-from-secret cookie-key)
                        #++ (parse-cookies-txt *ineco-web-cookie-file*)
                        :force-binary t
                        :parameters
                        `(("export_filter" . ,(format nil "17/06/2020;17/06/2030;~a;3" account))
                          ("export_sorting" . "")
                          ("DXScript" . "...")
                          ("DXCss" . "....")
                          ("DXMVCEditorsValues"
                           . ,(format nil "{\"export_filter\":\"17/06/2020;17/06/2030;~a;3\",\"export_sorting\":null}" account))
                          ("btnExportCsv" . "btnExportCsv")))
   :encoding :utf-16))

(defun load-all-statements-by-type (type cookie-key)
  (let ((accounts (or (rest (assoc type *ineco-accounts*))
                      (error "no accounts for type ~s" type))))
    (iter (for (name account) in accounts)
          (setf (gethash (cons type name) *loaded-statements*)
                (load-statement account cookie-key))
          (dbg "loaded: ~s ~s ~s" type name account))))

(defun write-statements ()
  (let ((dir (uiop:ensure-directory-pathname *ineco-web-statement-dir*)))
    (uiop:delete-directory-tree dir
                                :if-does-not-exist :ignore
                                :validate #'(lambda (path)
                                              (search "/statements/" (namestring path))))
    (uiop:ensure-all-directories-exist (list dir))
    (iter (for (key csv) in-hashtable *loaded-statements*)
          (let ((filename
                  (merge-pathnames (format nil "~(~a--~a~).csv" (car key) (cdr key))
                                   dir)))
            (dbg "Writing ~s" filename)
            (write-string-into-file (remove #\return csv) filename)))))

(defun load-all-statements ()
  (ineco-web-login :web-ep-cookies "Inecobank - EP")
  (load-all-statements-by-type :ep :web-ep-cookies)
  (ineco-web-login :web-personal-cookies "Inecobank - personal")
  (load-all-statements-by-type :personal :web-personal-cookies)
  (write-statements))

(defun get-statement-csv (key)
  (or (gethash key *loaded-statements*)
      (error "statement not found for key ~s" key)))

(defun read-web-statement (&optional (key '(:personal . :card-mc)))
  (validate-statement
   (fixup-statement
    (with-input-from-string (in (get-statement-csv key))
      (read-statement in)))))

sms.lisp

(in-package :yobabank)

;;;; SMS PARSING

(defparameter *chat-id* ".inecobank")
(defparameter *sms-db*
  (concat
   (trivial-shell:get-env-var "HOME")
   "/Library/Messages/chat.db"))

(defparameter *sms-code-query*
  "select
     m.date/1000000000 + strftime('%s', '2001-01-01') as ts,
     m.text
   from message m, chat_message_join cm, chat ch
   where
    lower(m.text) like '%security code is%' and
    m.rowid = cm.message_id and cm.chat_id = ch.rowid and
    ch.chat_identifier like '%INECOBANK%'
   order by m.date desc
   limit 1")

(defmacro with-chat-db (&body body)
  `(sqlite:with-open-database (db *sms-db*) ,@body))

(defun load-sms ()
  (with-chat-db
    (sqlite:execute-to-list/named db *sms-query* ":chat" *chat-id*)))

(defun load-last-sms-code ()
  (iter (for row in
             (with-chat-db
               (sqlite:execute-to-list db *sms-code-query*)))
        (destructuring-bind (ts text) row
          (with-match (code) (".*Security Code is (\\d{4}).*" text :case-insensitive-mode t)
            (return (values ts code))))
        (finally
            (return (values 0 nil)))))

(defun wait-for-sms-code (&key (attempts 60) (interval 1) (start-ts (load-last-sms-code)))
  (iter (repeat attempts)
        (multiple-value-bind (cur-ts code) (load-last-sms-code)
          (unless (= start-ts cur-ts)
            (return code)))
        (sleep interval)
        (finally (error "timed out waiting for sms"))))