Skip to content

Instantly share code, notes, and snippets.

@ryanrhymes
Created March 9, 2017 17:26
Show Gist options
  • Save ryanrhymes/0b2c272eab558a9491478be002fbf2c5 to your computer and use it in GitHub Desktop.
Save ryanrhymes/0b2c272eab558a9491478be002fbf2c5 to your computer and use it in GitHub Desktop.
re-format mnist data set
(* Test neural network on MNIST *)
#require "bitstring";;
#require "bitstring.syntax";;
open Bitstring
type t = { magic : int; items : int }
let dataset = "/Users/liang/owl_dataset/t10k-images-idx3-ubyte"
let extract_files () =
let bits = bitstring_of_file dataset in
let bitl = bitstring_length bits in
let s = takebits 64 bits in
let magic, items = bitmatch s with { magic : 32; items : 32 } -> magic, items in
let l = ref [] in
let o = ref [] in
let r = ref (dropbits 64 bits) in
let _ = match magic with
| 2049l -> (
print_endline "labels";
for i = 1 to Int32.to_int items do
let s = takebits 8 !r in
let x = bitmatch s with { label : 8 } -> label in
l := !l @ [ x ];
r := dropbits 8 !r;
done;
Owl_utils.marshal_to_file !l "mnist-test-labels_raw"
)
| 2051l -> (
print_endline "images";
let m, n = bitmatch !r with { m : 32; n : 32 } -> m |> Int32.to_int, n |> Int32.to_int in
let vl = m * n in
Printf.printf "%i : %i %i\n" (Int32.to_int items) m n;
r := dropbits 64 !r;
for i = 1 to (Int32.to_int items) * vl do
let s = takebits 8 !r in
let x = bitmatch s with { d : 8 } -> d in
l := !l @ [ x ];
r := dropbits 8 !r;
if List.length (!l) mod vl = 0 then (
Printf.printf "process vector %i\n" (List.length !o + 1); flush_all ();
let v = Vec.zeros vl in
List.iteri (fun i a -> Vec.set v i (a |> float_of_int)) !l;
o := !o @ [ v ];
l := []
)
done;
Owl_utils.marshal_to_file !o "mnist-test-images_raw"
)
| _ -> failwith "error: unrecognised file types"
in
()
;;
let _ =
print_endline "test MNIST"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment