diff --git a/dev/checkstyle/suppressions.xml b/dev/checkstyle/suppressions.xml
index 1daf743a34e..168d115fb63 100644
--- a/dev/checkstyle/suppressions.xml
+++ b/dev/checkstyle/suppressions.xml
@@ -169,6 +169,8 @@
+
+
diff --git a/src/test/java/org/apache/sysds/test/applications/nn/BaseTest.java b/src/test/java/org/apache/sysds/test/applications/nn/BaseTest.java
index b5312612b62..f877616f2f1 100644
--- a/src/test/java/org/apache/sysds/test/applications/nn/BaseTest.java
+++ b/src/test/java/org/apache/sysds/test/applications/nn/BaseTest.java
@@ -30,7 +30,7 @@
public abstract class BaseTest extends MLContextTestBase {
protected static final Log LOG = LogFactory.getLog(BaseTest.class.getName());
- private static final String ERROR_STRING = "ERROR:";
+ protected static final String ERROR_STRING = "ERROR:";
protected void run(String name) {
run(name, false);
diff --git a/src/test/java/org/apache/sysds/test/applications/nn/NNOptimTest.java b/src/test/java/org/apache/sysds/test/applications/nn/NNOptimTest.java
new file mode 100644
index 00000000000..22b16e3788a
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/applications/nn/NNOptimTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.applications.nn;
+
+import org.junit.Test;
+
+public class NNOptimTest extends TestFolder {
+
+ @Test
+ public void sgd() {
+ run("sgd.dml");
+ }
+
+ @Test
+ public void sgd_momentum() {
+ run("sgd_momentum.dml");
+ }
+
+ @Test
+ public void rmsprop() {
+ run("rmsprop.dml");
+ }
+
+ @Test
+ public void sgd_nesterov() {
+ run("sgd_nesterov.dml");
+ }
+
+ @Test
+ public void adagrad() {
+ run("adagrad.dml");
+ }
+
+ @Test
+ public void adam() {
+ run("adam.dml");
+ }
+
+ @Test
+ public void adamw() {
+ run("adamw.dml");
+ }
+
+ @Test
+ public void lars() {
+ run("lars.dml");
+ }
+
+ @Test
+ public void scaled_gd() {
+ run("scaled_gd.dml");
+ }
+
+ @Override
+ protected void run(String name) {
+ super.run("component/optim/" + name);
+ }
+}
diff --git a/src/test/java/org/apache/sysds/test/applications/nn/NNOptimizerMNISTTest.java b/src/test/java/org/apache/sysds/test/applications/nn/NNOptimizerMNISTTest.java
new file mode 100644
index 00000000000..128dde5e43a
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/applications/nn/NNOptimizerMNISTTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.applications.nn;
+
+import static org.apache.sysds.api.mlcontext.ScriptFactory.dmlFromFile;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.sysds.api.mlcontext.Script;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+// This test runs multiple epochs on a 1 hidden layer neural net
+// while verifying an increasing accuracy and decreasing loss per epoch.
+@RunWith(value = Parameterized.class)
+@net.jcip.annotations.NotThreadSafe
+public class NNOptimizerMNISTTest extends TestFolder {
+ /*
+ * To add new optimizer to this test, add an
+ * adapter to "src/test/scripts/applications/nn/component/optim/adapters/"
+ * and add it to the parameter Collection. If needed, adjust the
+ * current function interface or make variables adjustable via parameter.
+ */
+
+ // region: parameters
+
+ private final String optimizer;
+ private final List> scriptArgs;
+
+ public NNOptimizerMNISTTest(String optimizer, List> scriptArgs) {
+ this.optimizer = optimizer;
+ this.scriptArgs = scriptArgs;
+ }
+
+ @Parameters(name = "{0}")
+ public static Collection data() {
+ return Arrays.asList(new Object[][] {
+ {"adagrad", args()},
+ {"adam", args()},
+ {"adamw", args()},
+ {"lars", args("$lr", 0.1)},
+ {"rmsprop", args()},
+ {"sgd", args()},
+ {"sgd_momentum", args()},
+ {"sgd_nesterov", args()}
+ });
+ }
+
+ private static List> args(Object... args) {
+ if(args.length % 2 != 0)
+ throw new IllegalArgumentException("args must be given as name/value pairs.");
+
+ List> pairs = new ArrayList<>(args.length / 2);
+ for(int i = 0; i < args.length; i += 2) {
+ if(!(args[i] instanceof String))
+ throw new IllegalArgumentException("argnames must be strings.");
+ pairs.add(Pair.of((String) args[i], args[i + 1]));
+ }
+ return pairs;
+ }
+
+ // endregion
+
+ @Test
+ public void mnist_optimizer_test() {
+ this.inject_optimizer_adapter_module_and_run(this.optimizer, this.scriptArgs);
+ }
+
+ // injects the adapter from "src/test/scripts/applications/nn/component/optim/adapters/"
+ // and executes the script while looking out for errors.
+ private void inject_optimizer_adapter_module_and_run(String optimizer, List> scriptArgs) {
+ Script script = dmlFromFile(getBaseFilePath() + "component/optim/mnist_optimizer_check.dml");
+ String moduleImportStatement = String.format("source(\"src/test/scripts/applications/nn/component/optim/adapters/%s.dml\") as optimizer", optimizer);
+ String newScriptString = script.getScriptString().replaceFirst("(?m)^.*# INSERT ADAPTER-MODULE #.*$", moduleImportStatement);
+ script.setScriptString(newScriptString);
+ for(Pair arg : scriptArgs)
+ script.in(arg.getLeft(), arg.getRight());
+ String stdOut = executeAndCaptureStdOut(script).getRight();
+ assertTrue(stdOut, !stdOut.contains(BaseTest.ERROR_STRING));
+ }
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adagrad.dml b/src/test/scripts/applications/nn/component/optim/adagrad.dml
new file mode 100644
index 00000000000..25d5091a10f
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adagrad.dml
@@ -0,0 +1,172 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adagrad.dml") as adagrad
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_adagrad = function() {
+ lr = 0.1
+ epsilon = 1e-8
+
+ # the cache starts at zero, same shape as the parameters
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ c0 = adagrad::init(X)
+ test_util::check_all_close(c0, matrix(0, rows=2, cols=3), 1e-12)
+
+ # first step (cache=0): cache1 = dX^2 and X1 = X - lr*dX / (sqrt(cache1)+eps)
+ dX = matrix("0.5 -0.5 0.5 -1 1 -1", rows=2, cols=3)
+ [X1, c1] = adagrad::update(X, dX, lr, epsilon, c0)
+ test_util::check_all_close(c1, dX^2, 1e-12)
+ test_util::check_all_close(X1, X - lr*dX / (sqrt(dX^2) + epsilon), 1e-12)
+ ch::check_finite(X1, "adagrad/finite")
+
+ # numerical stability: with dX=0 and cache=0 the update divides through
+ # sqrt(0)+eps. The parameters must be unchanged, the cache untouched, and
+ # the result finite.
+ [Xz, cz] = adagrad::update(X, matrix(0, rows=2, cols=3), lr, epsilon, c0)
+ test_util::check_all_close(Xz, X, 1e-12)
+ test_util::check_all_close(cz, c0, 1e-12)
+ ch::check_finite(Xz, "adagrad/stability_zero_grad")
+
+ # the cache accumulates squared gradients across steps. The exact-equality
+ # check below implicitly proves non-decreasing and step-shrinking; no need
+ # for separate min(c2-c1)>=0 or step-2= 0. This is cheap and catches sign / subtraction-instead-of-addition bugs.
+ if (min(c2) < 0) {
+ test_util::fail("adagrad/non_negative: cache went negative")
+ }
+
+ # mixed-sign accumulation: c += dX^2 must not depend on the sign of dX.
+ # Running the same |dX| with alternating signs must produce the same cache
+ # as running it with constant signs. If someone wrote c += dX (or c += |dX|
+ # miscomputed), this asymmetry surfaces here — the constant-sign path would
+ # match the quadratic accumulator, the mixed-sign one would not.
+ Xms = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ cms = adagrad::init(Xms)
+ gpos = matrix("0.5 -0.5 0.5 -1 1 -1", rows=2, cols=3)
+ gneg = -gpos
+ [Xms, cms] = adagrad::update(Xms, gpos, lr, epsilon, cms)
+ [Xms, cms] = adagrad::update(Xms, gneg, lr, epsilon, cms)
+ # cache after two steps must equal gpos^2 + gneg^2 == 2 * gpos^2
+ test_util::check_all_close(cms, 2 * gpos^2, 1e-12)
+
+ # sign / direction sanity: at cache=0 with X=0, the step is opposite to dX.
+ # Catches a sign flip in the update that the exact-equality checks would
+ # miss if analytic + impl were flipped in lockstep.
+ Xs = matrix(0, rows=2, cols=3)
+ cs0 = adagrad::init(Xs)
+ [Xs1, cs1_out] = adagrad::update(Xs, dX, lr, epsilon, cs0)
+ if (sum((sign(Xs1) == -sign(dX)) | (dX == 0)) < length(dX)) {
+ test_util::fail("adagrad/sign: step is not opposite to the gradient")
+ }
+
+ # lr=0 no-op: with a zero learning rate the parameters must not move, though
+ # the cache should still accumulate (adagrad's cache update is independent
+ # of lr and needed to keep state consistent for later steps).
+ [Xlr0, clr0] = adagrad::update(X, dX, 0.0, epsilon, c0)
+ test_util::check_all_close(Xlr0, X, 1e-12)
+ test_util::check_all_close(clr0, dX^2, 1e-12)
+
+ # large-magnitude gradient: adagrad's sqrt(cache) normalization tames a huge
+ # dX to roughly a step of size lr (at cache=0 the update is
+ # lr*dX/(|dX|+eps) which for |dX| >> eps is ~lr*sign(dX)). The result must
+ # stay finite.
+ dXbig = matrix(1e10, rows=2, cols=3)
+ [Xbig, cbig] = adagrad::update(X, dXbig, lr, epsilon, c0)
+ ch::check_finite(Xbig, "adagrad/finite_large_grad")
+ if (max(abs(Xbig - X)) > lr + 1e-6) {
+ test_util::fail("adagrad/large_grad_step: step exceeded lr under a huge gradient")
+ }
+
+ # non-square shape: a 1xN parameter vector must round-trip through init
+ # and update without any broadcasting or shape confusion.
+ Xr = matrix("1 -2 3 -4 5", rows=1, cols=5)
+ dXr = matrix("0.5 -0.5 0.25 -0.25 0.1", rows=1, cols=5)
+ cr0 = adagrad::init(Xr)
+ test_util::check_all_close(cr0, matrix(0, rows=1, cols=5), 1e-12)
+ [Xr1, cr1] = adagrad::update(Xr, dXr, lr, epsilon, cr0)
+ test_util::check_all_close(cr1, dXr^2, 1e-12)
+ test_util::check_all_close(Xr1, Xr - lr*dXr / (sqrt(dXr^2) + epsilon), 1e-12)
+
+ # minimizing 0.5*||W||^2 (its gradient is just W) should drive the loss down
+ # to well below its starting value; adagrad needs a larger lr than sgd here
+ # because the cache damps the effective step size. Also verify the cache
+ # magnitude: with constant-sign gradients the cache after N steps should
+ # be a real multiple of the first squared-gradient contribution — not just
+ # the single-step value.
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ cache = adagrad::init(W)
+ init_loss = 0.5 * sum(W^2)
+ # snapshot the step-1 cache so the growth check below is anchored to a
+ # value from this run rather than a hard-coded magic number.
+ [W_snap, cache_snap] = adagrad::update(W, W, 0.5, epsilon, cache)
+ W = W_snap
+ cache = cache_snap
+ cache_step1 = cache
+ losses = matrix(0, rows=300, cols=1)
+ losses[1,] = 0.5 * sum(W^2)
+ for (i in 2:300) {
+ [W, cache] = adagrad::update(W, W, 0.5, epsilon, cache)
+ losses[i,] = 0.5 * sum(W^2)
+ }
+ # descent: on a convex quadratic with constant-sign gradients and this lr,
+ # adagrad is monotone-decreasing. Elsewhere this would be too strict — for
+ # this specific setup it is the right invariant.
+ ch::check_decreasing(losses, "adagrad/descent")
+ if (0.5 * sum(W^2) >= 0.01 * init_loss) {
+ test_util::fail("adagrad/converged: loss was not reduced enough")
+ }
+ # cache-growth check: anchor to the observed step-1 cache rather than a
+ # hard-coded number. Every element must have grown by at least 2x over 300
+ # steps; a leak / reset would collapse toward 1x and fail. The factor is
+ # conservative (not ~300x) because W shrinks toward zero, so later
+ # squared-gradient contributions are small.
+ if (min(cache / cache_step1) < 2.0) {
+ test_util::fail("adagrad/cache_grows: cache did not accumulate as expected")
+ }
+ # and cache must remain non-negative after all those updates
+ if (min(cache) < 0) {
+ test_util::fail("adagrad/cache_non_negative: cache went negative during training")
+ }
+
+ # non-quadratic loss: f(w) = sum(w^4), grad = 4*w^3. Unlike the quadratic
+ # bowl the gradient magnitude changes non-linearly with w, so a broken
+ # adaptive rescaling that only happens to work on ||W||^2 fails here.
+ Wq = matrix("1.5 -1.2 0.8 -0.9 1.1 -1.4", rows=2, cols=3)
+ cacheq = adagrad::init(Wq)
+ init_lossq = sum(Wq^4)
+ for (i in 1:1500) {
+ gq = 4 * Wq^3
+ [Wq, cacheq] = adagrad::update(Wq, gq, 0.5, epsilon, cacheq)
+ }
+ if (sum(Wq^4) >= 0.05 * init_lossq) {
+ test_util::fail("adagrad/converged_quartic: loss was not reduced enough")
+ }
+ ch::check_finite(Wq, "adagrad/converged_quartic")
+}
+
+test_adagrad()
diff --git a/src/test/scripts/applications/nn/component/optim/adam.dml b/src/test/scripts/applications/nn/component/optim/adam.dml
new file mode 100644
index 00000000000..a4df08f51b7
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adam.dml
@@ -0,0 +1,199 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adam.dml") as adam
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_adam = function() {
+ lr = 0.01
+ beta1 = 0.9
+ beta2 = 0.999
+ epsilon = 1e-8
+
+ # both moment vectors start at zero with the same shape as the parameters
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ [m0, v0] = adam::init(X)
+ test_util::check_all_close(m0, matrix(0, rows=2, cols=3), 1e-12)
+ test_util::check_all_close(v0, matrix(0, rows=2, cols=3), 1e-12)
+
+ # first step (m=v=0, t=0 -> internal t becomes 1):
+ # m1 = (1-beta1)*dX
+ # v1 = (1-beta2)*dX^2
+ # lr_eff = lr * sqrt(1-beta2) / (1-beta1)
+ # X1 = X - lr_eff * m1 / (sqrt(v1) + eps)
+ dX = matrix("0.5 -0.5 0.5 -1 1 -1", rows=2, cols=3)
+ [X1, m1, v1] = adam::update(X, dX, lr, beta1, beta2, epsilon, 0, m0, v0)
+ exp_m1 = (1-beta1)*dX
+ exp_v1 = (1-beta2)*dX^2
+ lr_eff1 = lr * sqrt(1-beta2) / (1-beta1)
+ exp_X1 = X - lr_eff1 * exp_m1 / (sqrt(exp_v1) + epsilon)
+ test_util::check_all_close(m1, exp_m1, 1e-12)
+ test_util::check_all_close(v1, exp_v1, 1e-12)
+ test_util::check_all_close(X1, exp_X1, 1e-12)
+ ch::check_finite(X1, "adam/finite")
+
+ # second step (t=1 -> internal t becomes 2, bias correction uses beta^2).
+ # expected values are chained from the previous *expected* values, not from
+ # the returned m1/v1: a persistent bug in the moment update that agrees with
+ # itself across steps must still fail the analytic check here.
+ [X2, m2, v2] = adam::update(X1, dX, lr, beta1, beta2, epsilon, 1, m1, v1)
+ exp_m2 = beta1*exp_m1 + (1-beta1)*dX
+ exp_v2 = beta2*exp_v1 + (1-beta2)*dX^2
+ lr_eff2 = lr * sqrt(1-beta2^2) / (1-beta1^2)
+ exp_X2 = X1 - lr_eff2 * exp_m2 / (sqrt(exp_v2) + epsilon)
+ test_util::check_all_close(m2, exp_m2, 1e-12)
+ test_util::check_all_close(v2, exp_v2, 1e-12)
+ test_util::check_all_close(X2, exp_X2, 1e-12)
+
+ # non-trivial timestep: exercise the bias-correction exponents at t=17 from
+ # arbitrary non-zero moment state. Small t values (1, 2) hide bugs like
+ # replacing beta^t with beta*t, which agrees at t=1 and is close at t=2.
+ m_seed = matrix("0.1 -0.2 0.3 -0.4 0.5 -0.6", rows=2, cols=3)
+ v_seed = matrix("0.01 0.02 0.03 0.04 0.05 0.06", rows=2, cols=3)
+ t_in = 16 # internal t becomes 17
+ [Xt, mt, vt] = adam::update(X, dX, lr, beta1, beta2, epsilon, t_in, m_seed, v_seed)
+ exp_mt = beta1*m_seed + (1-beta1)*dX
+ exp_vt = beta2*v_seed + (1-beta2)*dX^2
+ lr_efft = lr * sqrt(1-beta2^17) / (1-beta1^17)
+ exp_Xt = X - lr_efft * exp_mt / (sqrt(exp_vt) + epsilon)
+ test_util::check_all_close(mt, exp_mt, 1e-12)
+ test_util::check_all_close(vt, exp_vt, 1e-12)
+ test_util::check_all_close(Xt, exp_Xt, 1e-12)
+
+ # numerical stability: with dX=0 and m=v=0 the update divides through
+ # sqrt(0)+eps. The step must be exactly zero and the result finite.
+ [Xz, mz, vz] = adam::update(X, matrix(0, rows=2, cols=3), lr, beta1, beta2, epsilon, 0, m0, v0)
+ test_util::check_all_close(Xz, X, 1e-12)
+ test_util::check_all_close(mz, matrix(0, rows=2, cols=3), 1e-12)
+ test_util::check_all_close(vz, matrix(0, rows=2, cols=3), 1e-12)
+ ch::check_finite(Xz, "adam/stability_zero_grad")
+
+ # non-square shape: a 1xN parameter vector must round-trip through init and
+ # update without any broadcasting or shape confusion. lr_eff at t=1 is
+ # recomputed inline rather than reused from lr_eff1 above, so a shape bug
+ # cannot silently piggy-back on a value from the differently-shaped 2x3 case.
+ Xr = matrix("1 -2 3 -4 5", rows=1, cols=5)
+ dXr = matrix("0.5 -0.5 0.25 -0.25 0.1", rows=1, cols=5)
+ [mr0, vr0] = adam::init(Xr)
+ test_util::check_all_close(mr0, matrix(0, rows=1, cols=5), 1e-12)
+ [Xr1, mr1, vr1] = adam::update(Xr, dXr, lr, beta1, beta2, epsilon, 0, mr0, vr0)
+ exp_mr1 = (1-beta1)*dXr
+ exp_vr1 = (1-beta2)*dXr^2
+ lr_eff_r1 = lr * sqrt(1-beta2) / (1-beta1)
+ exp_Xr1 = Xr - lr_eff_r1 * exp_mr1 / (sqrt(exp_vr1) + epsilon)
+ test_util::check_all_close(Xr1, exp_Xr1, 1e-12)
+
+ # sign / direction sanity: at X=0 with m=v=0 the update sign must be opposite
+ # to dX, independent of the analytic formula. Catches a wrong-sign bug that
+ # the exp_* comparisons above would miss if analytic + impl were flipped
+ # in lockstep.
+ Xs = matrix(0, rows=2, cols=3)
+ [ms0, vs0] = adam::init(Xs)
+ [Xs1, ms1_out, vs1_out] = adam::update(Xs, dX, lr, beta1, beta2, epsilon, 0, ms0, vs0)
+ # every element of Xs1 must have opposite sign to dX (or be zero if dX is zero)
+ if (sum((sign(Xs1) == -sign(dX)) | (dX == 0)) < length(dX)) {
+ test_util::fail("adam/sign: step is not opposite to the gradient")
+ }
+
+ # state independence from X: m and v depend only on dX and prior (m, v),
+ # not on X. Two updates with the same gradient/state but different parameters
+ # must produce identical moment state.
+ Xa = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ Xb = matrix("10 -20 30 40 -50 60", rows=2, cols=3)
+ [Xa_out, ma1, va1] = adam::update(Xa, dX, lr, beta1, beta2, epsilon, 0, m0, v0)
+ [Xb_out, mb1, vb1] = adam::update(Xb, dX, lr, beta1, beta2, epsilon, 0, m0, v0)
+ test_util::check_all_close(ma1, mb1, 1e-12)
+ test_util::check_all_close(va1, vb1, 1e-12)
+
+ # bias-correction equivalence at t=2: the compact form used by the impl
+ # lr_eff = lr * sqrt(1-beta2^t)/(1-beta1^t); X -= lr_eff * m / (sqrt(v)+eps)
+ # must agree with the textbook form
+ # m_hat = m/(1-beta1^t); v_hat = v/(1-beta2^t); X -= lr * m_hat/(sqrt(v_hat)+eps)
+ # up to how epsilon is scaled inside the sqrt. Tolerance is loosened to 1e-4
+ # to absorb that epsilon placement (still >> eps=1e-8, so any exponentiation
+ # bug shows up).
+ [X2t, m2t, v2t] = adam::update(X1, dX, lr, beta1, beta2, epsilon, 1, m1, v1)
+ m_hat2 = exp_m2 / (1-beta1^2)
+ v_hat2 = exp_v2 / (1-beta2^2)
+ exp_X2_textbook = X1 - lr * m_hat2 / (sqrt(v_hat2) + epsilon)
+ test_util::check_all_close(X2t, exp_X2_textbook, 1e-4)
+
+ # lr=0 no-op: with a zero learning rate the parameters must not move at all.
+ # Moment state still updates because Adam decouples state accumulation from
+ # the parameter update; only X is pinned here.
+ [X0, m0lr, v0lr] = adam::update(X, dX, 0.0, beta1, beta2, epsilon, 0, m0, v0)
+ test_util::check_all_close(X0, X, 1e-12)
+
+ # large-magnitude gradient: Adam's adaptive scaling should tame a huge dX to
+ # roughly a step of size lr, and the result must stay finite (no overflow).
+ dXbig = matrix(1e10, rows=2, cols=3)
+ [Xbig, mbig, vbig] = adam::update(X, dXbig, lr, beta1, beta2, epsilon, 0, m0, v0)
+ ch::check_finite(Xbig, "adam/finite_large_grad")
+ # at t=1 the per-element step magnitude collapses to lr for dX >> eps
+ if (max(abs(Xbig - X)) > lr + 1e-6) {
+ test_util::fail("adam/large_grad_step: step exceeded lr under a huge gradient")
+ }
+
+ # minimizing 0.5*||W||^2 over 400 steps should reduce the loss to <1% of init.
+ # Adam can oscillate, so strict monotone descent would be wrong here; instead
+ # we assert progress at a checkpoint (loss@200 < loss@400_init) to catch a
+ # run that stalls after some initial progress.
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ [m, v] = adam::init(W)
+ init_loss = 0.5 * sum(W^2)
+ loss_mid = 0.0
+ t = 0
+ for (i in 1:400) {
+ [W, m, v] = adam::update(W, W, 0.05, beta1, beta2, epsilon, t, m, v)
+ t = t + 1
+ if (i == 200) {
+ loss_mid = 0.5 * sum(W^2)
+ }
+ }
+ final_loss = 0.5 * sum(W^2)
+ if (final_loss >= 0.01 * init_loss) {
+ test_util::fail("adam/converged: loss was not reduced enough")
+ }
+ # no-stall: the second half must also make progress, not just the first half
+ if (final_loss >= loss_mid) {
+ test_util::fail("adam/no_stall: loss did not decrease between step 200 and 400")
+ }
+
+ # non-quadratic loss: f(w) = sum(w^4), grad = 4*w^3. Unlike the quadratic bowl
+ # the gradient magnitude changes non-linearly with w, so a broken adaptive
+ # rescaling that only happens to work on ||W||^2 will fail here.
+ Wq = matrix("1.5 -1.2 0.8 -0.9 1.1 -1.4", rows=2, cols=3)
+ [mq, vq] = adam::init(Wq)
+ init_lossq = sum(Wq^4)
+ tq = 0
+ for (i in 1:600) {
+ gq = 4 * Wq^3
+ [Wq, mq, vq] = adam::update(Wq, gq, 0.05, beta1, beta2, epsilon, tq, mq, vq)
+ tq = tq + 1
+ }
+ if (sum(Wq^4) >= 0.01 * init_lossq) {
+ test_util::fail("adam/converged_quartic: loss was not reduced enough")
+ }
+ ch::check_finite(Wq, "adam/converged_quartic")
+}
+
+test_adam()
diff --git a/src/test/scripts/applications/nn/component/optim/adamw.dml b/src/test/scripts/applications/nn/component/optim/adamw.dml
new file mode 100644
index 00000000000..be117018997
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adamw.dml
@@ -0,0 +1,245 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adamw.dml") as adamw
+source("scripts/nn/optim/adam.dml") as adam
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_adamw = function() {
+ lr = 0.01
+ beta1 = 0.9
+ beta2 = 0.999
+ epsilon = 1e-8
+ lambda = 0.01
+
+ # both moment vectors start at zero with the same shape as the parameters
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ [m0, v0] = adamw::init(X)
+ test_util::check_all_close(m0, matrix(0, rows=2, cols=3), 1e-12)
+ test_util::check_all_close(v0, matrix(0, rows=2, cols=3), 1e-12)
+
+ # first step (m=v=0, t=0 -> internal t becomes 1):
+ # m1 = (1-beta1)*dX, v1 = (1-beta2)*dX^2
+ # m_hat = m1/(1-beta1), v_hat = v1/(1-beta2) (bias correction at t=1)
+ # X1 = X - lr * ( m_hat/(sqrt(v_hat)+eps) + lambda*X )
+ dX = matrix("0.5 -0.5 0.5 -1 1 -1", rows=2, cols=3)
+ [X1, m1, v1] = adamw::update(X, dX, lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ exp_m1 = (1-beta1)*dX
+ exp_v1 = (1-beta2)*dX^2
+ m_hat1 = exp_m1 / (1-beta1)
+ v_hat1 = exp_v1 / (1-beta2)
+ exp_X1 = X - lr * ( m_hat1 / (sqrt(v_hat1) + epsilon) + lambda*X )
+ test_util::check_all_close(m1, exp_m1, 1e-12)
+ test_util::check_all_close(v1, exp_v1, 1e-12)
+ test_util::check_all_close(X1, exp_X1, 1e-12)
+ ch::check_finite(X1, "adamw/finite")
+
+ # second step (t=1 -> internal t becomes 2). Expected values are chained
+ # from exp_m1/exp_v1, not m1/v1, so a persistent bug in the moment update
+ # that agrees with itself across steps must still fail this check.
+ [X2, m2, v2] = adamw::update(X1, dX, lr, beta1, beta2, epsilon, lambda, 1, m1, v1)
+ exp_m2 = beta1*exp_m1 + (1-beta1)*dX
+ exp_v2 = beta2*exp_v1 + (1-beta2)*dX^2
+ m_hat2 = exp_m2 / (1-beta1^2)
+ v_hat2 = exp_v2 / (1-beta2^2)
+ exp_X2 = X1 - lr * ( m_hat2 / (sqrt(v_hat2) + epsilon) + lambda*X1 )
+ test_util::check_all_close(m2, exp_m2, 1e-12)
+ test_util::check_all_close(v2, exp_v2, 1e-12)
+ test_util::check_all_close(X2, exp_X2, 1e-12)
+
+ # non-trivial timestep: exercise the bias-correction exponents at t=17 from
+ # arbitrary non-zero moment state. Small t values (1, 2) can hide bugs like
+ # replacing beta^t with beta*t which agrees at t=1 and is close at t=2.
+ m_seed = matrix("0.1 -0.2 0.3 -0.4 0.5 -0.6", rows=2, cols=3)
+ v_seed = matrix("0.01 0.02 0.03 0.04 0.05 0.06", rows=2, cols=3)
+ t_in = 16 # internal t becomes 17
+ [Xt, mt, vt] = adamw::update(X, dX, lr, beta1, beta2, epsilon, lambda, t_in, m_seed, v_seed)
+ exp_mt = beta1*m_seed + (1-beta1)*dX
+ exp_vt = beta2*v_seed + (1-beta2)*dX^2
+ m_hatt = exp_mt / (1-beta1^17)
+ v_hatt = exp_vt / (1-beta2^17)
+ exp_Xt = X - lr * ( m_hatt / (sqrt(v_hatt) + epsilon) + lambda*X )
+ test_util::check_all_close(mt, exp_mt, 1e-12)
+ test_util::check_all_close(vt, exp_vt, 1e-12)
+ test_util::check_all_close(Xt, exp_Xt, 1e-12)
+
+ # with a zero gradient only the decoupled weight-decay term fires:
+ # X_wd = X * (1 - lr*lambda)
+ [Xwd, mwd, vwd] = adamw::update(X, matrix(0, rows=2, cols=3), lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ test_util::check_all_close(Xwd, X * (1 - lr*lambda), 1e-12)
+
+ # numerical stability: with lambda=0 and dX=0 (v=0 too) the update divides
+ # through sqrt(0)+eps but must land exactly at X and be finite.
+ [Xnd, mnd, vnd] = adamw::update(X, matrix(0, rows=2, cols=3), lr, beta1, beta2, epsilon, 0.0, 0, m0, v0)
+ test_util::check_all_close(Xnd, X, 1e-12)
+ test_util::check_all_close(mnd, matrix(0, rows=2, cols=3), 1e-12)
+ test_util::check_all_close(vnd, matrix(0, rows=2, cols=3), 1e-12)
+ ch::check_finite(Xnd, "adamw/stability_zero_grad")
+
+ # decoupled weight decay: the adaptive-gradient part of the update must not
+ # depend on lambda. Running two updates with the same gradient/state but
+ # different lambdas must differ by exactly lr*(lambda_a - lambda_b)*X. If
+ # someone re-couples the decay (e.g. bakes it into sqrt(v_hat)) this fails.
+ lambda_a = 0.02
+ lambda_b = 0.001
+ [Xa, ma, va] = adamw::update(X, dX, lr, beta1, beta2, epsilon, lambda_a, 0, m0, v0)
+ [Xb, mb, vb] = adamw::update(X, dX, lr, beta1, beta2, epsilon, lambda_b, 0, m0, v0)
+ test_util::check_all_close(Xb - Xa, lr * (lambda_a - lambda_b) * X, 1e-12)
+ # the moment state must also be independent of lambda
+ test_util::check_all_close(ma, mb, 1e-12)
+ test_util::check_all_close(va, vb, 1e-12)
+
+ # per-element decay: at a coordinate where dX=0 but X!=0, the update must
+ # reduce to X*(1-lr*lambda) at that coordinate alone. Placing a zero in dX
+ # while other coordinates are non-zero exercises the branch that was only
+ # tested globally above.
+ #
+ # NOTE: the m_hat = dXmix / v_hat = dXmix^2 identities below hold only because
+ # this is the *first* step (t=1) with zero-initialized moments. At t>=2 or
+ # from non-zero m/v seed the analytic must go through the general formula.
+ dXmix = matrix("0.5 0 0.5 -1 0 -1", rows=2, cols=3)
+ [Xmix, mmix, vmix] = adamw::update(X, dXmix, lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ # analytic expected at t=1
+ m_hatm = dXmix # (1-b1)*dX / (1-b1)
+ # v_hat at dX=0 is 0, so sqrt(v_hat)+eps = eps and the term is 0/eps = 0
+ v_hatm = dXmix^2 # (1-b2)*dX^2 / (1-b2)
+ exp_Xmix = X - lr * ( m_hatm / (sqrt(v_hatm) + epsilon) + lambda*X )
+ test_util::check_all_close(Xmix, exp_Xmix, 1e-12)
+ # check the zero-gradient coordinates directly: they must move only by decay.
+ # This equality holds only because m,v were zero-initialized here — at a
+ # later step non-zero prior state would leave a residual adaptive term.
+ # (row 1 col 2 and row 2 col 2 are the zeros in dXmix)
+ test_util::check_all_close(Xmix[1,2], X[1,2] * (1 - lr*lambda), 1e-12)
+ test_util::check_all_close(Xmix[2,2], X[2,2] * (1 - lr*lambda), 1e-12)
+
+ # lambda=0 must reduce AdamW effectively to Adam. This is the strongest
+ # single invariant of the decoupled formulation: with no decay the two
+ # update rules are algebraically equivalent up to how epsilon sits inside
+ # the sqrt (Adam uses lr_eff = lr*sqrt(1-b2^t)/(1-b1^t) with epsilon added
+ # after sqrt(v), while AdamW divides by (1-b^t) inside then adds epsilon to
+ # sqrt(v_hat) — so the two forms differ only by an epsilon-scaling of order
+ # eps ~ 1e-8). The moment state has no epsilon in it and must match to
+ # 1e-12. Any refactor that couples lambda into the adaptive term (or forgets
+ # a *X when lambda>0) will break the X check by orders of magnitude, well
+ # above the 1e-4 slack.
+ [Xw0, mw0, vw0] = adamw::update(X, dX, lr, beta1, beta2, epsilon, 0.0, 0, m0, v0)
+ [Xa0, ma0, va0] = adam::update (X, dX, lr, beta1, beta2, epsilon, 0, m0, v0)
+ test_util::check_all_close(Xw0, Xa0, 1e-4)
+ test_util::check_all_close(mw0, ma0, 1e-12)
+ test_util::check_all_close(vw0, va0, 1e-12)
+
+ # sign / direction sanity: at X=0 with m=v=0 and lambda=0, the update sign is
+ # opposite to dX regardless of the analytic formula. With lambda=0 the decay
+ # term drops out, so this pins the adaptive term's sign directly.
+ Xs = matrix(0, rows=2, cols=3)
+ [ms0, vs0] = adamw::init(Xs)
+ [Xs1, ms1_out, vs1_out] = adamw::update(Xs, dX, lr, beta1, beta2, epsilon, 0.0, 0, ms0, vs0)
+ if (sum((sign(Xs1) == -sign(dX)) | (dX == 0)) < length(dX)) {
+ test_util::fail("adamw/sign: step is not opposite to the gradient")
+ }
+
+ # state independence from X: m and v depend only on dX and prior (m, v),
+ # not on X. Two updates with the same gradient/state but different parameters
+ # must produce identical moment state (this holds regardless of lambda).
+ Xstate_a = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ Xstate_b = matrix("10 -20 30 40 -50 60", rows=2, cols=3)
+ [Xsa, msa, vsa] = adamw::update(Xstate_a, dX, lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ [Xsb, msb, vsb] = adamw::update(Xstate_b, dX, lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ test_util::check_all_close(msa, msb, 1e-12)
+ test_util::check_all_close(vsa, vsb, 1e-12)
+
+ # lr=0 no-op: with a zero learning rate the parameters must not move, even
+ # when lambda>0 — the decay term is scaled by lr too, so both contributions
+ # vanish and X must be pinned exactly.
+ [Xlr0, mlr0, vlr0] = adamw::update(X, dX, 0.0, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ test_util::check_all_close(Xlr0, X, 1e-12)
+
+ # large-magnitude gradient: the adaptive part is scale-invariant, so the
+ # step from the gradient term stays ~lr. With lambda>0 the decay term adds
+ # lr*lambda*|X|, still bounded and finite.
+ dXbig = matrix(1e10, rows=2, cols=3)
+ [Xbig, mbig, vbig] = adamw::update(X, dXbig, lr, beta1, beta2, epsilon, lambda, 0, m0, v0)
+ ch::check_finite(Xbig, "adamw/finite_large_grad")
+ bound = lr + lr * lambda * max(abs(X)) + 1e-6
+ if (max(abs(Xbig - X)) > bound) {
+ test_util::fail("adamw/large_grad_step: step exceeded expected bound under huge gradient")
+ }
+
+ # non-square shape: a 1xN parameter vector must round-trip through init and
+ # update without any broadcasting or shape confusion.
+ Xr = matrix("1 -2 3 -4 5", rows=1, cols=5)
+ dXr = matrix("0.5 -0.5 0.25 -0.25 0.1", rows=1, cols=5)
+ [mr0, vr0] = adamw::init(Xr)
+ test_util::check_all_close(mr0, matrix(0, rows=1, cols=5), 1e-12)
+ [Xr1, mr1, vr1] = adamw::update(Xr, dXr, lr, beta1, beta2, epsilon, lambda, 0, mr0, vr0)
+ exp_Xr1 = Xr - lr * ( dXr / (sqrt(dXr^2) + epsilon) + lambda*Xr )
+ test_util::check_all_close(Xr1, exp_Xr1, 1e-12)
+
+ # minimizing 0.5*||W||^2 over 400 steps should reduce the loss to <1% of init
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ [m, v] = adamw::init(W)
+ init_loss = 0.5 * sum(W^2)
+ t = 0
+ for (i in 1:400) {
+ [W, m, v] = adamw::update(W, W, 0.05, beta1, beta2, epsilon, 0.001, t, m, v)
+ t = t + 1
+ }
+ if (0.5 * sum(W^2) >= 0.01 * init_loss) {
+ test_util::fail("adamw/converged: loss was not reduced enough")
+ }
+
+ # decay-heavy convergence: with lambda=0.1 the L2 penalty dominates and the
+ # optimizer must still terminate to a small W. This variant catches bugs
+ # that only surface when the decay term is a first-order contributor (the
+ # lambda=0.001 case above is basically Adam).
+ Wd = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ [md, vd] = adamw::init(Wd)
+ init_lossd = 0.5 * sum(Wd^2)
+ td = 0
+ for (i in 1:400) {
+ [Wd, md, vd] = adamw::update(Wd, Wd, 0.05, beta1, beta2, epsilon, 0.1, td, md, vd)
+ td = td + 1
+ }
+ if (0.5 * sum(Wd^2) >= 0.01 * init_lossd) {
+ test_util::fail("adamw/converged_decay_heavy: loss was not reduced enough")
+ }
+ ch::check_finite(Wd, "adamw/converged_decay_heavy")
+
+ # non-quadratic loss: f(w) = sum(w^4), grad = 4*w^3. Unlike the quadratic
+ # bowl the gradient magnitude changes non-linearly with w, so a broken
+ # adaptive rescaling that only happens to work on ||W||^2 fails here.
+ Wq = matrix("1.5 -1.2 0.8 -0.9 1.1 -1.4", rows=2, cols=3)
+ [mq, vq] = adamw::init(Wq)
+ init_lossq = sum(Wq^4)
+ tq = 0
+ for (i in 1:600) {
+ gq = 4 * Wq^3
+ [Wq, mq, vq] = adamw::update(Wq, gq, 0.05, beta1, beta2, epsilon, 0.001, tq, mq, vq)
+ tq = tq + 1
+ }
+ if (sum(Wq^4) >= 0.01 * init_lossq) {
+ test_util::fail("adamw/converged_quartic: loss was not reduced enough")
+ }
+ ch::check_finite(Wq, "adamw/converged_quartic")
+}
+
+test_adamw()
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/adagrad.dml b/src/test/scripts/applications/nn/component/optim/adapters/adagrad.dml
new file mode 100644
index 00000000000..c3d5d719db7
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/adagrad.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adagrad.dml") as adagrad
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ cache = adagrad::init(X)
+ state = list(cache)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ cache = as.matrix(state[1])
+ [X, cache] = adagrad::update(X, dX, lr, epsilon, cache)
+ state = list(cache)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/adam.dml b/src/test/scripts/applications/nn/component/optim/adapters/adam.dml
new file mode 100644
index 00000000000..b7795538fcf
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/adam.dml
@@ -0,0 +1,41 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adam.dml") as adam
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ [m, v] = adam::init(X)
+ t = 0
+ state = list(m, v, t)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ m = as.matrix(state[1])
+ v = as.matrix(state[2])
+ t = as.integer(as.scalar(state[3]))
+ [X, m, v] = adam::update(X, dX, lr, beta1, beta2, epsilon, t, m, v)
+ state = list(m, v, t + 1)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/adamw.dml b/src/test/scripts/applications/nn/component/optim/adapters/adamw.dml
new file mode 100644
index 00000000000..78c2e4bce1f
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/adamw.dml
@@ -0,0 +1,41 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/adamw.dml") as adamw
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ [m, v] = adamw::init(X)
+ t = 0
+ state = list(m, v, t)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ m = as.matrix(state[1])
+ v = as.matrix(state[2])
+ t = as.integer(as.scalar(state[3]))
+ [X, m, v] = adamw::update(X, dX, lr, beta1, beta2, epsilon, lambda, t, m, v)
+ state = list(m, v, t + 1)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/lars.dml b/src/test/scripts/applications/nn/component/optim/adapters/lars.dml
new file mode 100644
index 00000000000..a710c56b495
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/lars.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/lars.dml") as lars
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ v = lars::init(X)
+ state = list(v)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ v = as.matrix(state[1])
+ [X, v] = lars::update(X, dX, lr, mu, v, lambda, trust_coeff)
+ state = list(v)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/rmsprop.dml b/src/test/scripts/applications/nn/component/optim/adapters/rmsprop.dml
new file mode 100644
index 00000000000..7291ca25721
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/rmsprop.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/rmsprop.dml") as rmsprop
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ cache = rmsprop::init(X)
+ state = list(cache)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ cache = as.matrix(state[1])
+ [X, cache] = rmsprop::update(X, dX, lr, decay_rate, epsilon, cache)
+ state = list(cache)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/sgd.dml b/src/test/scripts/applications/nn/component/optim/adapters/sgd.dml
new file mode 100644
index 00000000000..d70b8621f02
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/sgd.dml
@@ -0,0 +1,36 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd.dml") as sgd
+
+# sgd doesn't need initialization. This is for interface conformity
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ state = list()
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ X = sgd::update(X, dX, lr)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/sgd_momentum.dml b/src/test/scripts/applications/nn/component/optim/adapters/sgd_momentum.dml
new file mode 100644
index 00000000000..5d17f522359
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/sgd_momentum.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd_momentum.dml") as sgd_momentum
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ v = sgd_momentum::init(X)
+ state = list(v)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ v = as.matrix(state[1])
+ [X, v] = sgd_momentum::update(X, dX, lr, mu, v)
+ state = list(v)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/adapters/sgd_nesterov.dml b/src/test/scripts/applications/nn/component/optim/adapters/sgd_nesterov.dml
new file mode 100644
index 00000000000..b5194e74633
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/adapters/sgd_nesterov.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd_nesterov.dml") as sgd_nesterov
+
+init_state = function(matrix[double] X)
+ return (list[unknown] state) {
+ v = sgd_nesterov::init(X)
+ state = list(v)
+}
+
+update_state = function(matrix[double] X, matrix[double] dX, double lr,
+ list[unknown] state, double mu, double decay_rate,
+ double epsilon, double beta1, double beta2,
+ double lambda, double trust_coeff)
+ return (matrix[double] X, list[unknown] state) {
+ v = as.matrix(state[1])
+ [X, v] = sgd_nesterov::update(X, dX, lr, mu, v)
+ state = list(v)
+}
diff --git a/src/test/scripts/applications/nn/component/optim/lars.dml b/src/test/scripts/applications/nn/component/optim/lars.dml
new file mode 100644
index 00000000000..858f2280418
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/lars.dml
@@ -0,0 +1,71 @@
+
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/lars.dml") as lars
+source("src/test/scripts/applications/nn/util.dml") as testutil
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as optim_check
+
+test_lars_init = function() {
+ #test velocity init all 0
+ W = rand(rows=4, cols=4, min=-1, max=1)
+ v = lars::init(W)
+ testutil::check_all_equal(v, matrix(0, rows=4, cols=4))
+}
+
+test_lars_global_learning_rate_fallback = function() {
+ # biases get unstable with lars.
+ # This LARS contains a fallback for the effective learning rate from local_lr*global_lr to global_lr only
+ b = rand(rows=4, cols=1, min=-1, max=1)
+ db = matrix(1, rows=4, cols=1)
+ v = lars::init(b)
+ lr = 0.1
+ mu = 0.0005
+ expected_b1 = b + (mu*v - lr*db)
+ [b1, v1] = lars::update(b, db, lr, 0, v, mu, 0.001)
+ testutil::check_all_close(b1, expected_b1, 1e-4)
+}
+
+test_lars_results_all_finite = function() {
+ M0 = matrix(0, rows=100, cols=100)
+ M1 = matrix(1, rows=100, cols=100)
+
+ lr = 0.1
+ mu = 0.0005
+
+ [X1, v1] = lars::update(M0, M0, lr, 0, M0, mu, 0.001)
+ [X2, v2] = lars::update(M1, M0, lr, 0, M0, mu, 0.001)
+ [X3, v3] = lars::update(M0, M1, lr, 0, M0, mu, 0.001)
+ [X4, v4] = lars::update(M1, M1, lr, 0, M0, mu, 0.001)
+
+ optim_check::check_finite(X1, "lars")
+ optim_check::check_finite(v1, "lars")
+ optim_check::check_finite(X2, "lars")
+ optim_check::check_finite(v2, "lars")
+ optim_check::check_finite(X3, "lars")
+ optim_check::check_finite(v3, "lars")
+ optim_check::check_finite(X4, "lars")
+ optim_check::check_finite(v4, "lars")
+}
+
+test_lars_init()
+test_lars_global_learning_rate_fallback()
+test_lars_results_all_finite()
diff --git a/src/test/scripts/applications/nn/component/optim/mnist_optimizer_check.dml b/src/test/scripts/applications/nn/component/optim/mnist_optimizer_check.dml
new file mode 100644
index 00000000000..b2cd7f2297b
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/mnist_optimizer_check.dml
@@ -0,0 +1,105 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("src/test/scripts/applications/nn/component/optim/adapters/sgd.dml") as optimizer # INSERT ADAPTER-MODULE #
+
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as optim_check
+
+source("scripts/nn/layers/affine.dml") as affine
+source("scripts/nn/layers/relu.dml") as relu
+source("scripts/nn/layers/softmax_cross_entropy_loss.dml") as softmax_cross_entropy_loss
+source("scripts/nn/layers/softmax.dml") as softmax
+
+input = 784
+hidden = 100
+output = 10
+
+lr = ifdef($lr, 0.001)
+
+epochs = 5
+batch_size = 100
+
+mnist_dataset = read("src/test/resources/datasets/MNIST/mnist_test.csv", format="csv", header=TRUE)
+mnist_dataset_len = nrow(mnist_dataset)
+
+mnist_labels = mnist_dataset[, 1]
+mnist_labels = table(seq(1, mnist_dataset_len), mnist_labels + 1, mnist_dataset_len, output)
+mnist_pixels = mnist_dataset[, 2:ncol(mnist_dataset)] / 255.0
+
+# 90% train-, 10% testdata
+n_train = floor(mnist_dataset_len * 0.9)
+n_test = mnist_dataset_len-n_train
+
+train_labels = mnist_labels[1:n_train, ]
+train_pixels = mnist_pixels[1:n_train, ]
+test_labels = mnist_labels[(n_train+1):mnist_dataset_len, ]
+test_pixels = mnist_pixels[(n_train+1):mnist_dataset_len, ]
+
+[W1, b1] = affine::init(input, hidden, 0) # set seed to -1 to use random seed
+[W2, b2] = affine::init(hidden, output, 1)
+
+state_W1 = optimizer::init_state(W1)
+state_b1 = optimizer::init_state(b1)
+state_W2 = optimizer::init_state(W2)
+state_b2 = optimizer::init_state(b2)
+
+num_iterations = ceil(n_train / batch_size)
+
+epoch_accuracies = matrix(0, rows=epochs, cols=1)
+epoch_losses = matrix(0, rows=epochs, cols=1)
+
+for (epoch in 1:epochs) {
+ for (i in seq(1, n_train, batch_size)) {
+ end_index = min(i+batch_size, n_train)
+ batch_labels = train_labels[i:end_index, ]
+ batch_pixels = train_pixels[i:end_index, ]
+
+ hidden_output_pre_activation = affine::forward(batch_pixels, W1, b1)
+ hidden_output = relu::forward(hidden_output_pre_activation)
+
+ prediction_pre_activation = affine::forward(hidden_output, W2, b2)
+ prediction = softmax::forward(prediction_pre_activation)
+
+ d_layer_2_pre_activation = softmax_cross_entropy_loss::backward(prediction_pre_activation, batch_labels)
+ [d_layer_1, d_W2, d_b2] = affine::backward(d_layer_2_pre_activation, hidden_output, W2, b2)
+
+ d_layer_1_pre_activation = relu::backward(d_layer_1, hidden_output_pre_activation)
+ [d_input, d_W1, d_b1] = affine::backward(d_layer_1_pre_activation, batch_pixels, W1, b1)
+
+ [W1, state_W1] = optimizer::update_state(W1, d_W1, lr, state_W1, 0.9, 0.9, 1e-5, 0.9, 0.999, 0.01, 0.001)
+ [b1, state_b1] = optimizer::update_state(b1, d_b1, lr, state_b1, 0.9, 0.9, 1e-5, 0.9, 0.999, 0.01, 0.001)
+ [W2, state_W2] = optimizer::update_state(W2, d_W2, lr, state_W2, 0.9, 0.9, 1e-5, 0.9, 0.999, 0.01, 0.001)
+ [b2, state_b2] = optimizer::update_state(b2, d_b2, lr, state_b2, 0.9, 0.9, 1e-5, 0.9, 0.999, 0.01, 0.001)
+ }
+
+ test_hidden_pre_activation = affine::forward(test_pixels, W1, b1)
+ test_hidden_output = relu::forward(test_hidden_pre_activation)
+ test_prediction_pre_activation = affine::forward(test_hidden_output, W2, b2)
+ test_prediction = softmax::forward(test_prediction_pre_activation)
+ test_loss = softmax_cross_entropy_loss::forward(test_prediction_pre_activation, test_labels)
+ test_acc = mean(rowIndexMax(test_prediction) == rowIndexMax(test_labels))
+
+ epoch_accuracies[epoch] = test_acc
+ epoch_losses[epoch] = test_loss
+}
+
+optim_check::check_increasing(epoch_accuracies, "epoch_accuracies")
+optim_check::check_decreasing(epoch_losses, "epoch_losses")
diff --git a/src/test/scripts/applications/nn/component/optim/optim_check.dml b/src/test/scripts/applications/nn/component/optim/optim_check.dml
new file mode 100644
index 00000000000..ce10e0ae466
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/optim_check.dml
@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+# Optimizer-specific checks that util.dml does not provide. Like the helpers in
+# util.dml, these stay quiet on success and print "ERROR:" on failure.
+
+check_finite = function(matrix[double] X, string name) {
+ n = length(X)
+ # NaN is never equal to itself, and the DBL_MAX bound catches +/-Inf
+ if (sum(X == X) < n | sum(abs(X) <= 1.7976931348623157e308) < n) {
+ print("ERROR: [" + name + "] contains a NaN or Inf")
+ }
+}
+
+check_decreasing = function(matrix[double] losses, string name) {
+ n = nrow(losses)
+ deltas = losses[2:n,] - losses[1:n-1,]
+ # NaN deltas are not < 0 either, so a blown-up run fails here too
+ if (sum(deltas < 0) < n - 1) {
+ print("ERROR: [" + name + "] loss is not strictly decreasing")
+ }
+}
+
+check_increasing = function(matrix[double] losses, string name) {
+ n = nrow(losses)
+ deltas = losses[2:n,] - losses[1:n-1,]
+ # NaN deltas are not > 0 either, so a blown-up run fails here too
+ if (sum(deltas > 0) < n - 1) {
+ print("ERROR: [" + name + "] loss is not strictly increasing")
+ }
+}
diff --git a/src/test/scripts/applications/nn/component/optim/rmsprop.dml b/src/test/scripts/applications/nn/component/optim/rmsprop.dml
new file mode 100644
index 00000000000..f0889c9fd79
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/rmsprop.dml
@@ -0,0 +1,69 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/rmsprop.dml") as rmsprop
+source("src/test/scripts/applications/nn/util.dml") as testutil
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as optim_check
+
+test_rmsprop_init = function() {
+ #test cache init all 0
+ W = rand(rows=4, cols=4, min=-1, max=1)
+ cache = rmsprop::init(W)
+ testutil::check_all_equal(cache, matrix(0, rows=4, cols=4))
+
+}
+
+test_rmsprop_epsilon_safeguard = function() {
+ # if cache = 0, decay_rate=1 and dX=matrix(0) ->
+ # Updated cache will be matrix(0).
+ # There should be no division by 0 because of the epsilon safeguard.
+ # All values should be finite (not NaN / infinite)
+ W = rand(rows=4, cols=4, min=-1, max=1)
+ cache = rmsprop::init(W)
+ dW = matrix(0, rows=4, cols=4)
+ [W, cache] = rmsprop::update(W, dW, 0.001, 1, 1e-4, cache)
+ optim_check::check_finite(W, "rmsprop updated parameters")
+ # updated cache should be 0
+ testutil::check_all_equal(cache, matrix(0, rows=4, cols=4))
+}
+
+test_rmsprop_single_update = function() {
+ W = rand(rows=4, cols=4, min=-1, max=1)
+ dW = rand(rows=4, cols=4, min=-1, max=1)
+
+ cache = rmsprop::init(W)
+
+ lr = 0.001
+ decay_rate = 0.9
+ epsilon = 1e-4
+
+ expected_cache = decay_rate*cache + (1-decay_rate)*dW^2
+ expected_W = W - (lr * dW / (sqrt(expected_cache)+epsilon))
+
+ [actual_W, actual_cache] = rmsprop::update(W, dW, lr, decay_rate, epsilon, cache)
+
+ testutil::check_all_equal(actual_cache, expected_cache)
+ testutil::check_all_equal(actual_W, expected_W)
+}
+
+test_rmsprop_init()
+test_rmsprop_epsilon_safeguard()
+test_rmsprop_single_update()
diff --git a/src/test/scripts/applications/nn/component/optim/scaled_gd.dml b/src/test/scripts/applications/nn/component/optim/scaled_gd.dml
new file mode 100644
index 00000000000..d3b43181039
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/scaled_gd.dml
@@ -0,0 +1,80 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/scaled_gd.dml") as scaled_gd
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+# scaled_gd keeps two factors X (m x r) and Y (n x r) instead of one parameter
+# matrix. its update() picks a random extension inside, so we can't check exact
+# numbers - instead we check the properties that must hold no matter the draw.
+test_scaled_gd = function() {
+ lr = 0.01
+ m = 6
+ n = 4
+ r = 2
+
+ # init just tells us the rank, which is the shared column count of X and Y
+ X = rand(rows=m, cols=r, min=-1, max=1)
+ Y = rand(rows=n, cols=r, min=-1, max=1)
+ r_init = scaled_gd::init(X, Y)
+ if (r_init != r) {
+ test_util::fail("scaled_gd/init_rank: expected r=" + r + ", got " + r_init)
+ }
+
+ # one step should hand back factors of the same shape it got
+ dX = rand(rows=m, cols=r, min=-1, max=1)
+ dY = rand(rows=n, cols=r, min=-1, max=1)
+ [X1, Y1] = scaled_gd::update(X, Y, dX, dY, lr, r)
+ if (nrow(X1) != m | ncol(X1) != r) {
+ test_util::fail("scaled_gd/shape_X: X_new must be " + m + "x" + r)
+ }
+ if (nrow(Y1) != n | ncol(Y1) != r) {
+ test_util::fail("scaled_gd/shape_Y: Y_new must be " + n + "x" + r)
+ }
+ ch::check_finite(X1, "scaled_gd/finite_X")
+ ch::check_finite(Y1, "scaled_gd/finite_Y")
+
+ # the update splits the kept singular values evenly, half into each factor,
+ # so the two factors end up balanced: X^t X and Y^t Y should match
+ GX = t(X1) %*% X1
+ GY = t(Y1) %*% Y1
+ test_util::check_all_close(GX, GY, 1e-8)
+
+ # and that matrix is just the singular values on the diagonal, so everything
+ # off the diagonal should be ~0
+ offdiag = GX - diag(diag(GX))
+ if (max(abs(offdiag)) > 1e-8) {
+ test_util::fail("scaled_gd/balanced_diagonal: X_new^t X_new is not diagonal")
+ }
+
+ # those diagonal entries are singular values, which can't be negative
+ if (min(diag(GX)) < -1e-12) {
+ test_util::fail("scaled_gd/nonneg_singvals: negative singular value retained")
+ }
+
+ # the result X * Y^t is low rank (rank r), so its total energy is just the
+ # sum of the kept singular values squared
+ M = X1 %*% t(Y1)
+ test_util::check_all_close(as.matrix(sum(M^2)), as.matrix(sum(diag(GX)^2)), 1e-6)
+}
+
+test_scaled_gd()
diff --git a/src/test/scripts/applications/nn/component/optim/sgd.dml b/src/test/scripts/applications/nn/component/optim/sgd.dml
new file mode 100644
index 00000000000..539390cc467
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/sgd.dml
@@ -0,0 +1,57 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd.dml") as sgd
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_sgd = function() {
+ lr = 0.1
+
+ # a single step just moves X against the gradient by lr
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ dX = matrix("0.5 0.5 0.5 1 1 1", rows=2, cols=3)
+ X1 = sgd::update(X, dX, lr)
+ test_util::check_all_close(X1, X - lr*dX, 1e-12)
+ ch::check_finite(X1, "sgd/finite")
+
+ # nothing moves without a gradient, and a zero learning rate is a no-op too
+ test_util::check_all_close(sgd::update(X, matrix(0, rows=2, cols=3), lr), X, 1e-12)
+ test_util::check_all_close(sgd::update(X, dX, 0.0), X, 1e-12)
+
+ # a single value works the same way: 5 - 0.1*2 = 4.8
+ small = sgd::update(matrix(5.0, rows=1, cols=1), matrix(2.0, rows=1, cols=1), lr)
+ test_util::check_all_close(small, matrix(4.8, rows=1, cols=1), 1e-12)
+
+ # minimizing 0.5*||W||^2 (its gradient is just W) should drive the loss to ~0
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ losses = matrix(0, rows=100, cols=1)
+ for (i in 1:100) {
+ W = sgd::update(W, W, lr)
+ losses[i,] = 0.5 * sum(W^2)
+ }
+ ch::check_decreasing(losses, "sgd/descent")
+ if (as.scalar(losses[100,]) >= 1e-6) {
+ test_util::fail("sgd/converged: final loss " + as.scalar(losses[100,]) + " not below 1e-6")
+ }
+}
+
+test_sgd()
diff --git a/src/test/scripts/applications/nn/component/optim/sgd_momentum.dml b/src/test/scripts/applications/nn/component/optim/sgd_momentum.dml
new file mode 100644
index 00000000000..97cf470ff8e
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/sgd_momentum.dml
@@ -0,0 +1,64 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd_momentum.dml") as sgd_momentum
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_sgd_momentum = function() {
+ lr = 0.1
+ mu = 0.9
+
+ # the velocity starts at zero, same shape as the parameters
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ v0 = sgd_momentum::init(X)
+ test_util::check_all_close(v0, matrix(0, rows=2, cols=3), 1e-12)
+
+ # first step (v=0): the velocity becomes -lr*g and the params move by it
+ g = matrix("0.5 0.5 0.5 1 1 1", rows=2, cols=3)
+ [X1, v1] = sgd_momentum::update(X, g, lr, mu, v0)
+ test_util::check_all_close(v1, -lr*g, 1e-12)
+ test_util::check_all_close(X1, X - lr*g, 1e-12)
+ ch::check_finite(X1, "sgd_momentum/finite")
+
+ # second step carries the old velocity, so with a constant gradient
+ # X lands at X - lr*g*(2+mu)
+ [X2, v2] = sgd_momentum::update(X1, g, lr, mu, v1)
+ test_util::check_all_close(X2, X - lr*g*(2+mu), 1e-12)
+
+ # that build-up means step two moves further than step one
+ if (max(abs(X2 - X1)) <= max(abs(X1 - X))) {
+ test_util::fail("sgd_momentum/accumulates: second step did not move further than the first")
+ }
+
+ # with a little momentum it still drives 0.5*||W||^2 down to ~0
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ v = sgd_momentum::init(W)
+ init_loss = 0.5 * sum(W^2)
+ for (i in 1:100) {
+ [W, v] = sgd_momentum::update(W, W, 0.05, 0.5, v)
+ }
+ if (0.5 * sum(W^2) >= 0.01 * init_loss) {
+ test_util::fail("sgd_momentum/converged: loss was not reduced enough")
+ }
+}
+
+test_sgd_momentum()
diff --git a/src/test/scripts/applications/nn/component/optim/sgd_nesterov.dml b/src/test/scripts/applications/nn/component/optim/sgd_nesterov.dml
new file mode 100644
index 00000000000..cfae19982b6
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/optim/sgd_nesterov.dml
@@ -0,0 +1,61 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/optim/sgd_nesterov.dml") as sgd_nesterov
+source("scripts/nn/optim/sgd_momentum.dml") as sgd_momentum
+source("src/test/scripts/applications/nn/util.dml") as test_util
+source("src/test/scripts/applications/nn/component/optim/optim_check.dml") as ch
+
+test_sgd_nesterov = function() {
+ lr = 0.1
+ mu = 0.9
+
+ # the velocity starts at zero, same shape as the parameters
+ X = matrix("1 2 3 4 5 6", rows=2, cols=3)
+ v0 = sgd_nesterov::init(X)
+ test_util::check_all_close(v0, matrix(0, rows=2, cols=3), 1e-12)
+
+ # first step (v=0): velocity is -lr*g, and the look-ahead lands X at X - (1+mu)*lr*g
+ g = matrix("0.5 0.5 0.5 1 1 1", rows=2, cols=3)
+ [X1, v1] = sgd_nesterov::update(X, g, lr, mu, v0)
+ test_util::check_all_close(v1, -lr*g, 1e-12)
+ test_util::check_all_close(X1, X - (1+mu)*lr*g, 1e-12)
+ ch::check_finite(X1, "sgd_nesterov/finite")
+
+ # the look-ahead makes it differ from plain momentum given the same inputs
+ [Xm, vm] = sgd_momentum::update(X, g, lr, mu, v0)
+ if (max(abs(X1 - Xm)) <= 1e-9) {
+ test_util::fail("sgd_nesterov/differs: update matched plain momentum")
+ }
+
+ # it still drives 0.5*||W||^2 down to ~0
+ W = matrix("3 -4 5 -6 7 -8", rows=2, cols=3)
+ v = sgd_nesterov::init(W)
+ init_loss = 0.5 * sum(W^2)
+ for (i in 1:100) {
+ [W, v] = sgd_nesterov::update(W, W, 0.05, 0.5, v)
+ }
+ if (0.5 * sum(W^2) >= 0.01 * init_loss) {
+ test_util::fail("sgd_nesterov/converged: loss was not reduced enough")
+ }
+}
+
+test_sgd_nesterov()