diff --git a/README.md b/README.md
index c33f7932..20d34f61 100644
--- a/README.md
+++ b/README.md
@@ -29,13 +29,12 @@ Read the paper [here](https://arxiv.org/abs/1902.06714).
|------------|------------------|------------------------|----------------------|--------------|---------------|
| Input | `input` | n/a | 1, 3 | n/a | n/a |
| Dense (fully-connected) | `dense` | `input1d`, `flatten` | 1 | ✅ | ✅ |
-| Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ❌ |
+| Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅(*) |
| Max-pooling (2-d) | `maxpool2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅ |
| Flatten | `flatten` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ |
| Reshape (1-d to 3-d) | `reshape` | `input1d`, `dense`, `flatten` | 3 | ✅ | ✅ |
-**Note:** The training of convolutional layers has been discovered to be broken
-as of release 0.13.0. This will be fixed in a future (hopefully next) release.
+(*) See Issue [#145](https://github.com/modern-fortran/neural-fortran/issues/145) regarding non-converging CNN training on the MNIST dataset.
## Getting started
diff --git a/src/nf/nf_layer_constructors_submodule.f90 b/src/nf/nf_layer_constructors_submodule.f90
index 002a83ba..91de36ed 100644
--- a/src/nf/nf_layer_constructors_submodule.f90
+++ b/src/nf/nf_layer_constructors_submodule.f90
@@ -8,7 +8,7 @@
use nf_input3d_layer, only: input3d_layer
use nf_maxpool2d_layer, only: maxpool2d_layer
use nf_reshape_layer, only: reshape3d_layer
- use nf_activation, only: activation_function, sigmoid
+ use nf_activation, only: activation_function, relu, sigmoid
implicit none
@@ -27,7 +27,7 @@ pure module function conv2d(filters, kernel_size, activation) result(res)
if (present(activation)) then
allocate(activation_tmp, source=activation)
else
- allocate(activation_tmp, source=sigmoid())
+ allocate(activation_tmp, source=relu())
end if
res % activation = activation_tmp % get_name()
diff --git a/src/nf/nf_network_submodule.f90 b/src/nf/nf_network_submodule.f90
index 5bafb7cf..a35064bd 100644
--- a/src/nf/nf_network_submodule.f90
+++ b/src/nf/nf_network_submodule.f90
@@ -305,12 +305,14 @@ pure module subroutine backward(self, output)
select type(next_layer => self % layers(n + 1) % p)
type is(dense_layer)
call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
- type is(flatten_layer)
- call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
type is(conv2d_layer)
call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
+ type is(flatten_layer)
+ call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
type is(maxpool2d_layer)
call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
+ type is(reshape3d_layer)
+ call self % layers(n) % backward(self % layers(n - 1), next_layer % gradient)
end select
end if
diff --git a/test/test_conv2d_layer.f90 b/test/test_conv2d_layer.f90
index ae0474ee..5840063b 100644
--- a/test/test_conv2d_layer.f90
+++ b/test/test_conv2d_layer.f90
@@ -24,9 +24,9 @@ program test_conv2d_layer
write(stderr, '(a)') 'conv2d layer should not be marked as initialized yet.. failed'
end if
- if (.not. conv_layer % activation == 'sigmoid') then
+ if (.not. conv_layer % activation == 'relu') then
ok = .false.
- write(stderr, '(a)') 'conv2d layer is defaults to sigmoid activation.. failed'
+ write(stderr, '(a)') 'conv2d layer defaults to relu activation.. failed'
end if
input_layer = input([3, 32, 32])
@@ -62,7 +62,7 @@ program test_conv2d_layer
call conv_layer % forward(input_layer)
call conv_layer % get_output(output)
- if (.not. all(abs(output - 0.5) < tolerance)) then
+ if (.not. all(abs(output) < tolerance)) then
ok = .false.
write(stderr, '(a)') 'conv2d layer with zero input and sigmoid function must forward to all 0.5.. failed'
end if
diff --git a/test/test_conv2d_network.f90 b/test/test_conv2d_network.f90
index 4d87a520..47c9a819 100644
--- a/test/test_conv2d_network.f90
+++ b/test/test_conv2d_network.f90
@@ -1,7 +1,7 @@
program test_conv2d_network
use iso_fortran_env, only: stderr => error_unit
- use nf, only: conv2d, input, network
+ use nf, only: conv2d, input, network, dense, sgd, maxpool2d
implicit none
@@ -21,6 +21,7 @@ program test_conv2d_network
ok = .false.
end if
+ ! Test for output shape
allocate(sample_input(3, 32, 32))
sample_input = 0
@@ -32,6 +33,115 @@ program test_conv2d_network
ok = .false.
end if
+ deallocate(sample_input, output)
+
+ training1: block
+
+ type(network) :: cnn
+ real :: y(1)
+ real :: tolerance = 1e-5
+ integer :: n
+ integer, parameter :: num_iterations = 1000
+
+ ! Test training of a minimal constant mapping
+ allocate(sample_input(1, 5, 5))
+ call random_number(sample_input)
+
+ cnn = network([ &
+ input(shape(sample_input)), &
+ conv2d(filters=1, kernel_size=3), &
+ conv2d(filters=1, kernel_size=3), &
+ dense(1) &
+ ])
+
+ y = [0.1234567]
+
+ do n = 1, num_iterations
+ call cnn % forward(sample_input)
+ call cnn % backward(y)
+ call cnn % update(optimizer=sgd(learning_rate=1.))
+ if (all(abs(cnn % predict(sample_input) - y) < tolerance)) exit
+ end do
+
+ if (.not. n <= num_iterations) then
+ write(stderr, '(a)') &
+ 'convolutional network 1 should converge in simple training.. failed'
+ ok = .false.
+ end if
+
+ end block training1
+
+ training2: block
+
+ type(network) :: cnn
+ real :: x(1, 8, 8)
+ real :: y(1)
+ real :: tolerance = 1e-5
+ integer :: n
+ integer, parameter :: num_iterations = 1000
+
+ call random_number(x)
+ y = [0.1234567]
+
+ cnn = network([ &
+ input(shape(x)), &
+ conv2d(filters=1, kernel_size=3), &
+ maxpool2d(pool_size=2), &
+ conv2d(filters=1, kernel_size=3), &
+ dense(1) &
+ ])
+
+ do n = 1, num_iterations
+ call cnn % forward(x)
+ call cnn % backward(y)
+ call cnn % update(optimizer=sgd(learning_rate=1.))
+ if (all(abs(cnn % predict(x) - y) < tolerance)) exit
+ end do
+
+ if (.not. n <= num_iterations) then
+ write(stderr, '(a)') &
+ 'convolutional network 2 should converge in simple training.. failed'
+ ok = .false.
+ end if
+
+ end block training2
+
+ training3: block
+
+ type(network) :: cnn
+ real :: x(1, 12, 12)
+ real :: y(9)
+ real :: tolerance = 1e-5
+ integer :: n
+ integer, parameter :: num_iterations = 5000
+
+ call random_number(x)
+ y = [0.12345, 0.23456, 0.34567, 0.45678, 0.56789, 0.67890, 0.78901, 0.89012, 0.90123]
+
+ cnn = network([ &
+ input(shape(x)), &
+ conv2d(filters=1, kernel_size=3), & ! 1x12x12 input, 1x10x10 output
+ maxpool2d(pool_size=2), & ! 1x10x10 input, 1x5x5 output
+ conv2d(filters=1, kernel_size=3), & ! 1x5x5 input, 1x3x3 output
+ dense(9) & ! 9 outputs
+ ])
+
+ do n = 1, num_iterations
+ call cnn % forward(x)
+ call cnn % backward(y)
+ call cnn % update(optimizer=sgd(learning_rate=1.))
+ if (all(abs(cnn % predict(x) - y) < tolerance)) exit
+ end do
+
+ if (.not. n <= num_iterations) then
+ write(stderr, '(a)') &
+ 'convolutional network 3 should converge in simple training.. failed'
+ ok = .false.
+ end if
+
+ end block training3
+
+
if (ok) then
print '(a)', 'test_conv2d_network: All tests passed.'
else