re-format mnist data set

mnist.ml

(* Test neural network on MNIST *)

#require "bitstring";;
#require "bitstring.syntax";;

open Bitstring

type t = { magic : int; items : int }

let dataset = "/Users/liang/owl_dataset/t10k-images-idx3-ubyte"

let extract_files () =
  let bits = bitstring_of_file dataset in
  let bitl = bitstring_length bits in
  let s = takebits 64 bits in
  let magic, items = bitmatch s with { magic : 32; items : 32 } -> magic, items in
  let l = ref [] in
  let o = ref [] in
  let r = ref (dropbits 64 bits) in
  let _ = match magic with
  | 2049l -> (
      print_endline "labels";
      for i = 1 to Int32.to_int items do
        let s = takebits 8 !r in
        let x = bitmatch s with { label : 8 } -> label in
        l := !l @ [ x ];
        r := dropbits 8 !r;
      done;
      Owl_utils.marshal_to_file !l "mnist-test-labels_raw"
    )
  | 2051l -> (
      print_endline "images";
      let m, n = bitmatch !r with { m : 32; n : 32 } -> m |> Int32.to_int, n |> Int32.to_int in
      let vl = m * n in
      Printf.printf "%i : %i %i\n" (Int32.to_int items) m n;
      r := dropbits 64 !r;
      for i = 1 to (Int32.to_int items) * vl do
        let s = takebits 8 !r in
        let x = bitmatch s with { d : 8 } -> d in
        l := !l @ [ x ];
        r := dropbits 8 !r;
        if List.length (!l) mod vl = 0 then (
          Printf.printf "process vector %i\n" (List.length !o + 1); flush_all ();
          let v = Vec.zeros vl in
          List.iteri (fun i a -> Vec.set v i (a |> float_of_int)) !l;
          o := !o @ [ v ];
          l := []
        )
      done;
      Owl_utils.marshal_to_file !o "mnist-test-images_raw"
    )
  | _ -> failwith "error: unrecognised file types"
  in
  ()
;;

let _ =
  print_endline "test MNIST"