Last active
September 23, 2018 15:27
-
-
Save cashiwamochi/7a938761c37c8493f2f7afe9f3b45d34 to your computer and use it in GitHub Desktop.
This nim source converts binary of cifar10 to PNG. Some packages are needed, if you use this, please install them for yourself.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import streams | |
import sequtils | |
import nimPNG, math, tables, base64 | |
import private.buffer | |
import progress | |
const | |
height: int = 32 | |
width: int = 32 | |
type | |
cifarFileNameArray = array[0..5, string] | |
patchData = object | |
id: uint8 | |
seq_r_values: seq[uint8] | |
seq_g_values: seq[uint8] | |
seq_b_values: seq[uint8] | |
Image = ref object | |
data: string | |
width, height: int | |
colorType: PNGcolorType | |
bitDepth: int | |
patchPngData = object | |
id: uint8 | |
image: Image | |
proc checkCifarExist(cifar_file_array: cifarFileNameArray) : bool = | |
for file_name in cifar_file_array: | |
if not existsFile(file_name): | |
return false | |
return true | |
proc loadCifar( cifar_file: string ): seq[patchData] = | |
echo "|| ", cifar_file, " is being loaded ... " | |
result = newSeq[patchData](10000) | |
const num_data_in_row = 3073 | |
const max_row_num = 10000 | |
var row_idx : int = 0 | |
var col_idx : int = 0 | |
var fs = newFileStream(cifar_file, fmRead) | |
var id : uint8 | |
var b = newSeq[uint8](1024) | |
var g = newSeq[uint8](1024) | |
var r = newSeq[uint8](1024) | |
if not isNil(fs): | |
while not fs.atEnd: | |
let value : uint8 = fs.readUint8() | |
if col_idx == 0: | |
id = value | |
elif col_idx <= 1024: | |
b[col_idx-1] = value | |
elif col_idx <= 2048: | |
g[col_idx-1-1024] = value | |
elif col_idx <= 3072: | |
r[col_idx-1-1024-1024] = value | |
else: | |
echo "|| [ERROR] Ploblem in parsing" | |
break | |
col_idx = col_idx + 1 | |
if col_idx == num_data_in_row: | |
col_idx = 0 | |
result[row_idx] = patchData(id: id, seq_b_values: b, seq_g_values: g, seq_r_values: r) | |
row_idx = row_idx + 1 | |
fs.close() | |
assert row_idx == max_row_num | |
proc encodePNG(data: seq[patchData]) : seq[patchPngData] = | |
result = newSeq[patchPngData](len(data)) | |
for i in 0..len(data)-1: | |
var image: Image | |
new(image) | |
image.width = width | |
image.height = height | |
image.colorType = LCT_RGBA | |
image.bitDepth = 8 | |
image.data = newString(width * height * 4) | |
for v in 0 .. height-1: | |
for u in 0 .. width-1: | |
image.data[v*width*4 + u*4 + 0] = chr(data[i].seq_b_values[v*width + u]) | |
image.data[v*width*4 + u*4 + 1] = chr(data[i].seq_g_values[v*width + u]) | |
image.data[v*width*4 + u*4 + 2] = chr(data[i].seq_r_values[v*width + u]) | |
image.data[v*width*4 + u*4 + 3] = chr(255) | |
result[i].image = image | |
result[i].id = data[i].id | |
# main | |
echo "||================================= Cifar-10 (Nim) ================================= " | |
let | |
root_path : string = os.getAppDir() | |
var | |
cifar_files: cifarFileNameArray | |
seq_patch_data : seq[patchData] # -> nil | |
seq_patch_data = @[] # necessary initialization | |
cifar_files = [root_path&"/cifar10/data_batch_1.bin", | |
root_path&"/cifar10/data_batch_2.bin", | |
root_path&"/cifar10/data_batch_3.bin", | |
root_path&"/cifar10/data_batch_4.bin", | |
root_path&"/cifar10/data_batch_5.bin", | |
root_path&"/cifar10/test_batch.bin"] | |
if not checkCifarExist(cifar_files): | |
quit(1) | |
createDir(root_path & "/png/train") | |
createDir(root_path & "/png/test") | |
for i in 0 .. 9: | |
createDir(root_path & "/png/train/" & intToStr(i,1)) | |
createDir(root_path & "/png/test/" & intToStr(i,1)) | |
var train_data = newSeq[patchData](50000) | |
for i in 1 .. 5: | |
train_data[(i-1)*10000 .. (i)*10000 - 1] = loadCifar(cifar_files[i-1]) | |
var test_png_data = encodePNG(loadCifar(cifar_files[len(cifar_files)-1])) | |
var train_png_data = encodePNG(train_data) | |
var bar = newProgressBar(total = 60000) | |
bar.start() | |
for idx, png in train_png_data: | |
discard savePNG32(root_path & "/png/train/" & intToStr(int(png.id)) & "/" & intToStr(idx) & ".png", | |
png.image.data, png.image.width, png.image.height) | |
bar.increment() | |
for idx, png in test_png_data: | |
discard savePNG32(root_path & "/png/test/" & intToStr(int(png.id)) & "/" & intToStr(idx) & ".png", | |
png.image.data, png.image.width, png.image.height) | |
bar.increment() | |
bar.finish() | |
echo "|| Converting has been done ... " |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment