fix nightly CI failure (#15452)

* fix nightly

* update with nosetest and fixed seed

* remove line

Author: roywei

ci/docker/runtime_functions.sh

@@ -1403,10 +1403,8 @@ nightly_estimator() {
     set -ex
     cd /work/mxnet/tests/nightly/estimator
     export PYTHONPATH=/work/mxnet/python/
-    python test_estimator_cnn.py --type gpu
-    python test_sentiment_rnn.py --type gpu
-    python test_estimator_cnn.py --type cpu
-    python test_sentiment_rnn.py --type cpu
+    nosetests test_estimator_cnn.py
+    nosetests test_sentiment_rnn.py
 }
 
 # Deploy

docs/tutorials/amp/amp_tutorial.md

@@ -35,6 +35,7 @@ import os
 import logging
 import warnings
 import time
+import numpy as np
 import mxnet as mx
 import mxnet.gluon as gluon
 from mxnet import autograd

tests/nightly/estimator/test_estimator_cnn.py

@@ -17,14 +17,22 @@
 
 # Test gluon estimator on CNN models
 
-import argparse
-import numpy as np
+import os
+import sys
+
 import mxnet as mx
+import numpy as np
 from mxnet import gluon, init, nd
 from mxnet.gluon import data
 from mxnet.gluon.contrib.estimator import estimator
 from mxnet.gluon.model_zoo import vision
 
+# use with_seed decorator in python/unittest/common.py
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'python', 'unittest'))
+from common import with_seed
+import unittest
+
+
 def load_data_mnist(batch_size, resize=None, num_workers=4):
     '''
     Load MNIST dataset
@@ -44,6 +52,7 @@ def load_data_mnist(batch_size, resize=None, num_workers=4):
         num_workers=num_workers)
     return train_iter, test_iter
 
+
 def bilinear_kernel(in_channels, out_channels, kernel_size):
     '''
     Bilinear interpolation using transposed convolution
@@ -60,6 +69,7 @@ def bilinear_kernel(in_channels, out_channels, kernel_size):
     weight[range(in_channels), range(out_channels), :, :] = filt
     return nd.array(weight)
 
+
 def get_net(model_name, context):
     if model_name == 'FCN':
         num_classes = 21
@@ -82,6 +92,8 @@ def get_net(model_name, context):
         loss_axis = -1
     return net, input_shape, label_shape, loss_axis
 
+
+@with_seed()
 def test_estimator_cpu():
     '''
     Test estimator by doing one pass over each model with synthetic data
@@ -112,6 +124,10 @@ def test_estimator_cpu():
                 val_data=val_data,
                 epochs=1)
 
+
+# using fixed seed to reduce flakiness in accuracy assertion
+@with_seed(7)
+@unittest.skipIf(mx.context.num_gpus() < 1, "skip if no GPU")
 def test_estimator_gpu():
     '''
     Test estimator by training resnet18_v1 for 5 epochs on MNIST and verify accuracy
@@ -139,13 +155,7 @@ def test_estimator_gpu():
 
     assert acc.get()[1] > 0.80
 
+
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='test gluon estimator')
-    parser.add_argument('--type', type=str, default='cpu')
-    opt = parser.parse_args()
-    if opt.type == 'cpu':
-        test_estimator_cpu()
-    elif opt.type == 'gpu':
-        test_estimator_gpu()
-    else:
-        raise RuntimeError("Unknown test type")
+    import nose
+    nose.runmodule()

tests/nightly/estimator/test_sentiment_rnn.py

@@ -20,17 +20,23 @@
 https://github.com/d2l-ai/d2l-en/blob/master/chapter_natural-language-processing/sentiment-analysis-rnn.md
 https://github.com/d2l-ai/d2l-en/blob/master/chapter_natural-language-processing/sentiment-analysis-cnn.md"""
 
-import argparse
+import collections
 import os
-import tarfile
 import random
-import collections
+import sys
+import tarfile
+
 import mxnet as mx
 from mxnet import nd, gluon
 from mxnet.contrib import text
 from mxnet.gluon import nn, rnn
 from mxnet.gluon.contrib.estimator import estimator
 
+# use with_seed decorator in python/unittest/common.py
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'python', 'unittest'))
+from common import with_seed
+import unittest
+
 
 class TextCNN(nn.Block):
     def __init__(self, vocab, embed_size, kernel_sizes, num_channels,
@@ -175,14 +181,10 @@ def pad(x):
     return features, labels
 
 
-def run(net, train_dataloader, test_dataloader, **kwargs):
+def run(net, train_dataloader, test_dataloader, num_epochs, ctx, lr):
     '''
     Train a test sentiment model
     '''
-    num_epochs = kwargs['epochs']
-    ctx = kwargs['ctx']
-    batch_size = kwargs['batch_size']
-    lr = kwargs['lr']
 
     # Define trainer
     trainer = mx.gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr})
@@ -199,14 +201,17 @@ def run(net, train_dataloader, test_dataloader, **kwargs):
     return acc
 
 
-def test_estimator_cpu(**kwargs):
+@with_seed()
+def test_estimator_cpu():
     '''
     Test estimator by doing one pass over each model with synthetic data
     '''
     models = ['TextCNN', 'BiRNN']
-    ctx = kwargs['ctx']
-    batch_size = kwargs['batch_size']
-    embed_size = kwargs['embed_size']
+    ctx = mx.cpu()
+    batch_size = 64
+    embed_size = 100
+    lr = 1
+    num_epochs = 1
 
     train_data = mx.nd.random.randint(low=0, high=100, shape=(2 * batch_size, 500))
     train_label = mx.nd.random.randint(low=0, high=2, shape=(2 * batch_size,))
@@ -229,18 +234,22 @@ def test_estimator_cpu(**kwargs):
             net = BiRNN(vocab_list, embed_size, num_hiddens, num_layers)
         net.initialize(mx.init.Xavier(), ctx=ctx)
 
-        run(net, train_dataloader, val_dataloader, **kwargs)
+        run(net, train_dataloader, val_dataloader, num_epochs=num_epochs, ctx=ctx, lr=lr)
 
 
-def test_estimator_gpu(**kwargs):
+# using fixed seed to reduce flakiness in accuracy assertion
+@with_seed(7)
+@unittest.skipIf(mx.context.num_gpus() < 1, "skip if no GPU")
+def test_estimator_gpu():
     '''
     Test estimator by training Bidirectional RNN for 5 epochs on the IMDB dataset
     and verify accuracy
     '''
-    ctx = kwargs['ctx']
-    batch_size = kwargs['batch_size']
-    num_epochs = kwargs['epochs']
-    embed_size = kwargs['embed_size']
+    ctx = mx.gpu(0)
+    batch_size = 64
+    num_epochs = 5
+    embed_size = 100
+    lr = 0.01
 
     # data
     download_imdb()
@@ -263,27 +272,11 @@ def test_estimator_gpu(**kwargs):
     net.embedding.weight.set_data(glove_embedding.idx_to_vec)
     net.embedding.collect_params().setattr('grad_req', 'null')
 
-    acc = run(net, train_dataloader, test_dataloader, **kwargs)
+    acc = run(net, train_dataloader, test_dataloader, num_epochs=num_epochs, ctx=ctx, lr=lr)
 
     assert acc.get()[1] > 0.70
 
 
-parser = argparse.ArgumentParser(description='test gluon estimator')
-parser.add_argument('--type', type=str, default='cpu')
-opt = parser.parse_args()
-kwargs = {
-    'batch_size': 64,
-    'lr': 0.01,
-    'embed_size': 100
-}
-
-if opt.type == 'cpu':
-    kwargs['ctx'] = mx.cpu()
-    kwargs['epochs'] = 1
-    test_estimator_cpu(**kwargs)
-elif opt.type == 'gpu':
-    kwargs['ctx'] = mx.gpu()
-    kwargs['epochs'] = 5
-    test_estimator_gpu(**kwargs)
-else:
-    raise RuntimeError("Unknown test type")
+if __name__ == '__main__':
+    import nose
+    nose.runmodule()