Update functions

src/Merlin.jl

@@ -47,7 +47,7 @@ end
 include("functions/activation.jl")
 include("functions/argmax.jl")
 include("functions/blas.jl")
-include("functions/cat.jl")
+include("functions/concat.jl")
 include("functions/conv.jl")
 include("functions/dropout.jl")
 include("functions/embeddings.jl")

src/functions/cat.jl → src/functions/concat.jl

@@ -1,24 +1,24 @@
-import Base.cat
+export concat
 
 """
-    cat(dim::Int, xs::Var...)
+    concat(dim::Int, xs::Var...)
 
 Concatenate arrays over the given dimension.
 
 # Example
 ```julia
 x1 = Var(rand(Float32,4,3))
 x2 = Var(rand(Float32,4,5))
-y = cat(2, x1, x2)
+y = concat(2, x1, x2)
 ```
 """
-function cat(dim::Int, xs::Var...)
+function concat(dim::Int, xs::Var...)
     y = cat(dim, map(x -> x.data, xs)...)
-    Var(y, cat, (dim,xs...))
+    Var(y, concat, (dim,xs...))
 end
-cat(dim::Int, xs::Node...; name="") = Node(cat, (dim,xs...), name)
+concat(dim::Int, xs::Node...; name="") = Node(concat, (dim,xs...), name)
 
-function addgrad!(y::Var, ::typeof(cat), dim::Int, xs::Var...)
+function addgrad!(y::Var, ::typeof(concat), dim::Int, xs::Var...)
     T, N = eltype(y), ndims(y)
     offset = 1
     for x in xs

src/functions/embeddings.jl

@@ -1,23 +1,16 @@
 export embeddings, lookup
 
-function embeddings(::Type{T}, insize::Int, outsize::Int; init_w=Normal(0,0.01)) where T
-    w = init_w(T, outsize, insize)
-    [zerograd(w[:,i]) for i=1:size(w,2)]
+function embeddings(::Type{T}, insize::Int, outsize::Int; init_W=Normal(0,0.01)) where T
+    W = init_W(T, outsize, insize)
+    [zerograd(W[:,i]) for i=1:size(W,2)]
 end
 
 function lookup(embeds::Vector{Var}, x::Var)
     y = lookup(embeds, x.data)
     xs = map(i -> embeds[i], vec(x.data))
     Var(y, lookup, (xs,))
 end
-
-function lookup(w::Var, x::Var)
-    @assert w.grad == nothing
-    y = lookup(w.data, x.data)
-    Var(y, lookup, ())
-end
-lookup(embeds::Vector{Var}, x::Node; name="") = Node(lookup, (embeds,x), name)
-lookup(w::Var, x::Node; name="") = Node(lookup, (w,x), name)
+lookup(embeds::Node, x::Node; name="") = Node(lookup, (embeds,x), name)
 
 function lookup(embeds::Vector{Var}, x::Array{Int})
     e1 = embeds[1].data

src/functions/linear.jl

@@ -29,13 +29,14 @@ function Linear(::Type{T}, insize::Int, outsize::Int; init_W=Xavier(), init_b=Fi
     b = init_b(T, outsize)
     Linear(zerograd(W), zerograd(b))
 end
-(f::Linear)(x) = linear(x, f.W, f.b)
+(f::Linear)(x::Var) = linear(x, f.W, f.b)
+(f::Linear)(x::Node) = linear(x, Node(f.W), Node(f.b))
 
 function linear(x::Var, W::Var, b::Var)
     y = linear(x.data, W.data, b.data)
     Var(y, linear, (x,W,b))
 end
-linear(x::Node, W::Var, b::Var; name="") = Node(linear, (x,W,b), name)
+linear(x::Node, W::Node, b::Node; name="") = Node(linear, (x,W,b), name)
 
 function linear(x::Matrix, W::Matrix, b)
     y = BLAS.gemm('T', 'N', W, x)

src/functions/recurrent.jl

@@ -19,14 +19,14 @@ function recurrent(f, x::Var, batchdims::Vector{Int}, h0::Var; rev=false)
             i += rev ? batchdims[p]-t : t-1
             push!(xts, x[:,i:i])
         end
-        xt = cat(2, xts...)
+        xt = concat(2, xts...)
         if size(h,2) < size(xt,2)
             @assert size(h,2) == 1
-            h = cat(2, ntuple(_ -> h, size(xt,2))...)
+            h = concat(2, ntuple(_ -> h, size(xt,2))...)
         elseif size(h,2) > size(xt,2)
             h = h[:,1:size(xt,2)]
         end
-        xt = cat(1, xt, h)
+        xt = concat(1, xt, h)
         h = f(xt)
         for j = 1:length(perm)
             p = perm[j]
@@ -36,7 +36,7 @@ function recurrent(f, x::Var, batchdims::Vector{Int}, h0::Var; rev=false)
             hs[i] = h[:,j:j]
         end
     end
-    cat(2, hs...)
+    concat(2, hs...)
 end
 
 doc"""
@@ -85,7 +85,6 @@ function LSTM(::Type{T}, insize::Int, outsize::Int; init_W=Xavier(), init_U=Orth
     U = init_U(T, insize, 4outsize)
     WU = cat(1, W, U)
     b = zeros(T, 4outsize)
-    b[1:outsize] = ones(T, outsize) # forget gate initializes to 1
     h0 = zeros(T, outsize, 1)
     c0 = zeros(T, outsize, 1)
     LSTM(zerograd(WU), zerograd(b), zerograd(h0), zerograd(c0))
@@ -102,7 +101,7 @@ function (lstm::LSTM)(x::Var, batchdims; rev=false)
         o = sigmoid(a[2n+1:3n,:])
         if size(c,2) < size(xt,2)
             @assert size(c,2) == 1
-            c = cat(2, ntuple(_ -> c, size(xt,2))...)
+            c = concat(2, ntuple(_ -> c, size(xt,2))...)
         elseif size(c,2) > size(xt,2)
             c = c[:,1:size(xt,2)]
         end
@@ -134,5 +133,5 @@ end
 function (bilstm::BiLSTM)(x::Var, batchdims)
     h1 = bilstm.fwd(x, batchdims)
     h2 = bilstm.bwd(x, batchdims, rev=true)
-    cat(1, h1, h2)
+    concat(1, h1, h2)
 end

src/graph.jl

@@ -7,7 +7,7 @@ mutable struct Node
     id::Int
 end
 
-Node(name::String) = Node(nothing, (), name)
+Node(f=nothing; name="") = Node(f, (), name)
 Node(f, args, name) = Node(f, args, name, 0)
 
 struct Graph
@@ -43,7 +43,7 @@ function (g::Graph)(xs::Pair...)
     for i = 1:length(g.nodes)
         node = g.nodes[i]
         if isempty(node.args)
-            isassigned(temps,i) || (temps[i] = node)
+            isassigned(temps,i) || (temps[i] = node.f)
         else
             args = map(node.args) do arg
                 isa(arg,Node) ? temps[arg.id] : arg

test/functions.jl

@@ -25,11 +25,11 @@ end
     @testgrad BLAS.gemv('T',1,A,x) A x
 end
 
-@testset "cat" for i = 1:5
+@testset "concat" for i = 1:5
     x1 = zerograd(randn(T,10,5,2))
     x2 = zerograd(randn(T,10,5,2))
     for dim = 1:3
-        @testgrad cat(dim,x1,x2) x1 x2
+        @testgrad concat(dim,x1,x2) x1 x2
     end
 end