conv fixes

denizyuret · Jul 4, 2020 · a2cffd7 · a2cffd7
1 parent 37987ba
commit a2cffd7
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 18 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -1,5 +1,33 @@
+Knet v1.3.6 Release Notes
+=========================
+
+* conv performance improvements.
+* doc fix and KnetArray method for nll, fixing #563.
+* Fix #561: Loading Knet breaks Julia's copyto! on 1.5.
+* Added update!(::Param,::Nothing) method to catch 0 gradient updates.
+* Added bessel support to GPU.
+* Further notes on GPU tools, especially for win (RocketRoss)
+* Fix #558: progress and minimize get size(x,d...) methods to support collect with Julia 1.4.
+* removed CUDAapi 1.0, 2.0 from Project.toml, not compat any more.
+* removed julia 1.0 (fails) and 1.3 (redundant) from travis testing.
+* added KnetArray(::CuArray) converter with shared pointer.
+* windows test fixes and install doc updates.
+* updated to work with CUDAapi v4.0 (iuliancioarca)
+* add gpu install docs for azure/ubuntu18.04 (Jan Weidner)
+
+Knet v1.3.5 Release Notes
+=========================
+4dd257a 2020-03-29
+
+* CI fixes.
+* cuda_visible_devices fix.
+* Warn when trying KnetArray without GPU.
+* Julia 1.4 compatibility fixes.
+
+
 Knet v1.3.4 Release Notes
 =========================
+8c50f62 2020-02-29
 
 * Tutorial notebook fixes.
 

diff --git a/src/conv.jl b/src/conv.jl
@@ -193,7 +193,7 @@ function pool(x::AbstractArray{T,N}; handle=nothing,
     pdims = PoolDims(x, window; padding = padding, stride = stride)
     y = (mode == 0 ? maxpool(x, pdims) :
          mode == 1 ? meanpool(x, pdims) :
-         mode == 2 ? meanpool(x, pdims) :
+         # mode == 2 ? meanpool(x, pdims) : ## CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING is missing in NNlib Issue #218
          error("mode=$mode is not supported for CPU pool."))
     alpha == 1 ? y : lmul!(alpha, y)
 end
@@ -209,15 +209,17 @@ function poolx(x::AbstractArray{T,N},y::AbstractArray{T,N},dy::AbstractArray{T,N
     pdims = PoolDims(x, window; padding = padding, stride = stride)
     dx = (mode == 0 ? ∇maxpool(dy, y, x, pdims) :
           mode == 1 ? ∇meanpool(dy, y, x, pdims) :
-          mode == 2 ? ∇meanpool(dy, y, x, pdims) :
+          # mode == 2 ? ∇meanpool(dy, y, x, pdims) : ## CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING is missing in NNlib Issue #218
           error("mode=$mode is not supported for CPU pool."))
     alpha == 1 ? dx : lmul!(alpha, dx)
 end
 
 
 """
 
-Unpooling; `reverse` of pooling.
+Unpooling; `reverse` of pooling. 
+
+TODO: Does not work correctly for every window, padding, mode combination. Test before use.
 
     x == pool(unpool(x;o...); o...)
 
@@ -598,7 +600,7 @@ end
 
 # This assumes one workspace per gpu. TODO: What about streams/contexts/handles etc?
 const CUDNN_WORKSPACE = []
-function cudnnWorkSpace(len=0;dev=gpu())
+function cudnnWorkSpace_fast_but_risky(len=0;dev=gpu())
     global CUDNN_WORKSPACE
     if dev==-1; error("No cudnnWorkSpace for CPU"); end
     i = dev+2
@@ -610,4 +612,4 @@ function cudnnWorkSpace(len=0;dev=gpu())
 end
 
 # Fresh workspace for every op is safer:
-cudnnWorkSpace_safe(len=0)=KnetArray{UInt8}(undef,len)
+cudnnWorkSpace(len=0)=KnetArray{UInt8}(undef,len)
diff --git a/test/conv.jl b/test/conv.jl
@@ -28,8 +28,6 @@ struct M370; layer; end;
         kw5 = KnetArray(aw5)
     end
 
-    @warn "CPU convolution tests temporarily turned off."
-#=
     @testset "cpuconv" begin
         ### Default
         @test gradcheck(pool, ax)
@@ -92,11 +90,11 @@ struct M370; layer; end;
         @test gradcheck(conv41, (aw,ax); rtol=TOL, kw=[(:mode,1),(:padding,1)])
         @test gradcheck(deconv41, (ad,ax); rtol=TOL, kw=[(:mode,1),(:padding,1)])
 
-        ### mode=2 (only for pool)
-        @test gradcheck(pool, ax; kw=[(:mode,2),(:padding,1)])
-        @test gradcheck(unpool, ax; kw=[(:mode,2),(:padding,1)])
-        @test isapprox(pool(unpool(ax;mode=2);mode=2),ax)
-        @test_broken isapprox(pool(unpool(ax;mode=2,padding=1);mode=2,padding=1),ax)
+        ### mode=2 (only for pool) -- is not supported in NNlib #218
+        # @test gradcheck(pool, ax; kw=[(:mode,2),(:padding,1)])
+        # @test gradcheck(unpool, ax; kw=[(:mode,2),(:padding,1)])
+        # @test isapprox(pool(unpool(ax;mode=2);mode=2),ax)
+        # @test isapprox(pool(unpool(ax;mode=2,padding=1);mode=2,padding=1),ax)
 
         ### alpha=2 (default=1)
         @test gradcheck(pool, ax; kw=[(:alpha,2)])
@@ -109,7 +107,6 @@ struct M370; layer; end;
         @test gradcheck(conv41, (aw,ax); rtol=TOL, kw=[(:alpha,2)])
         @test gradcheck(deconv41, (ad,ax); rtol=TOL, kw=[(:alpha,2)])
     end
-=#
 
     if gpu() >= 0; @testset "gpuconv" begin
         ### Default
@@ -201,9 +198,9 @@ struct M370; layer; end;
         @test gradcheck(deconv41, (kd,kx); rtol=TOL, kw=[(:mode,1),(:padding,1)])
 
         ### mode=2 (only for pool)
-        @test_broken isapprox(pool(kx;mode=2,padding=1), pool(ax;mode=2,padding=1))
+        # @test isapprox(pool(kx;mode=2,padding=1), pool(ax;mode=2,padding=1)) ## mode=2 is not supported in NNlib #218.
         @test gradcheck(pool, kx; kw=[(:mode,2),(:padding,1)])
-        @test isapprox(unpool(kx;mode=2,padding=1), unpool(ax;mode=2,padding=1))
+        # @test isapprox(unpool(kx;mode=2,padding=1), unpool(ax;mode=2,padding=1))  ## mode=2 is not supported in NNlib #218.
         @test gradcheck(unpool, kx; kw=[(:mode,2),(:padding,1)])
 
         ### alpha=2 (default=1)
@@ -215,10 +212,12 @@ struct M370; layer; end;
         @test gradcheck(pool, kx; kw=[(:alpha,2),(:mode,1),(:padding,1)])
         @test isapprox(unpool(kx;alpha=2,mode=1,padding=1), unpool(ax;alpha=2,mode=1,padding=1))
         @test gradcheck(unpool, kx; kw=[(:alpha,2),(:mode,1),(:padding,1)])
-        @test_broken isapprox(pool(kx;alpha=2,mode=2,padding=1), pool(ax;alpha=2,mode=2,padding=1))
-        @test gradcheck(pool, kx; kw=[(:alpha,2),(:mode,2),(:padding,1)])
-        @test isapprox(unpool(kx;alpha=2,mode=2,padding=1), unpool(ax;alpha=2,mode=2,padding=1))
+
+        # @test isapprox(pool(kx;alpha=2,mode=2,padding=1), pool(ax;alpha=2,mode=2,padding=1)) ## mode=2 is not supported in NNlib #218.
+        @test gradcheck(pool, kx; kw=[(:alpha,2),(:mode,2),(:padding,1)]) 
+        # @test isapprox(unpool(kx;alpha=2,mode=2,padding=1), unpool(ax;alpha=2,mode=2,padding=1)) ## mode=2 is not supported in NNlib #218.
         @test gradcheck(unpool, kx; kw=[(:alpha,2),(:mode,2),(:padding,1)])
+
         @test isapprox(conv4(kw,kx;alpha=2), conv4(aw,ax;alpha=2))
         @test gradcheck(conv41, (kw,kx); rtol=TOL, kw=[(:alpha,2)])
         @test isapprox(deconv4(kd,kx;alpha=2), deconv4(ad,ax;alpha=2))