| """ | |
| Micro-dali JIT Plan: | |
| - contains gemm, operator fusion, elementwise/reduction ops. | |
| - supports tensordot | |
| - supports 'jit' | |
| - supports conversion from gemm + im2col to conv2d (NHWC) | |
| - supports 'optimization' passes | |
| - supports 'implementation' registries for specialization | |
| (e.g. int vs float) |
| import datetime | |
| import linecache | |
| import os | |
| import pynvml3 | |
| import torch | |
| print_tensor_sizes = True | |
| last_tensor_sizes = set() | |
| gpu_profile_fn = f'{datetime.datetime.now():%d-%b-%y-%H:%M:%S}-gpu_mem_prof.txt' |
| import torch | |
| from torch.autograd import Variable | |
| leaves = [Variable(torch.zeros(5, 5), requires_grad=True) for _ in range(10)] | |
| intermediates = [l + i for i, l in enumerate(leaves)] | |
| loss = sum(v * i for i, v in enumerate(intermediates)).sum() | |
| # define a helper for dividing intermediates into groups | |
| def group(l, group_size): | |
| """Groups l into chunks of size group_size. |
| using Distributions | |
| import ForwardDiff: Dual, value, partials | |
| @generated function get_values(a::NTuple{N}) where {N} | |
| return ForwardDiff.tupexpr(i -> :(value(a[$i])),N) | |
| end | |
| ForwardDiff.value(p::ForwardDiff.Partials) = | |
| ForwardDiff.Partials(get_values(p.values)) |
Currently the AbstractArray type hierarchy has three major subtype trees:
In addition, we have the StridedArray typealias, which effectively “adds” strided SubArrays and ReshapedArrays as pseudo-subtypes of DenseArrays.
We also have the IndexStyle trait.
| from torch import nn | |
| from torch.autograd import Variable | |
| import torch | |
| l = nn.Linear(5,5).cuda() | |
| pl = nn.DataParallel(l) | |
| print("Checkpoint 1") | |
| a = Variable(torch.rand(5,5).cuda(), requires_grad=True) | |
| print("Checkpoint 2") | |
| print(pl(a)) # Here it gets stuck |
| def Rop(y, x, v): | |
| """Computes an Rop. | |
| Arguments: | |
| y (Variable): output of differentiated function | |
| x (Variable): differentiated input | |
| v (Variable): vector to be multiplied with Jacobian from the right | |
| """ | |
| w = torch.ones_like(y, requires_grad=True) | |
| return torch.autograd.grad(torch.autograd.grad(y, x, w), w, v) |
| # To ensure correct alignment (e.g. for an 80-bit type) | |
| struct StrWrap{T} | |
| value::T | |
| end | |
| function unsafe_reinterpret(T, a::A) where {A} | |
| if sizeof(T) <= sizeof(A) | |
| r = Ref(a) | |
| Base.@gc_preserve r begin | |
| u = convert(Ptr{T}, Base.unsafe_convert(Ptr{A}, r)) |
