1import theano
2import theano.tensor as T
3from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
4import numpy as np
5from load import mnist
6
7srng = RandomStreams()

 1def floatX(X):
 2    return np.asarray(X, dtype=theano.config.floatX)
 3
 4def init_weights(shape):
 5    return theano.shared(floatX(np.random.randn(*shape) * 0.01))
 6
 7def rectify(X):
 8    return T.maximum(X, 0.)
 9
10def softmax(X):
11    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))
12    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')
13
14def dropout(X, p=0.):
15    if p > 0:
16        retain_prob = 1 - p
17        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)
18        X /= retain_prob
19    return X
20
21def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):
22    grads = T.grad(cost=cost, wrt=params)
23    updates = []
24    for p, g in zip(params, grads):
25        acc = theano.shared(p.get_value() * 0.)
26        acc_new = rho * acc + (1 - rho) * g ** 2
27        gradient_scaling = T.sqrt(acc_new + epsilon)
28        g = g / gradient_scaling
29        updates.append((acc, acc_new))
30        updates.append((p, p - lr * g))
31    return updates

1from theano.tensor.nnet.conv import conv2d
2from theano.tensor.signal.downsample import max_pool_2d

1trX, teX, trY, teY = mnist(onehot=True)
2
3trX = trX.reshape(-1, 1, 28, 28)
4teX = teX.reshape(-1, 1, 28, 28)

 1def model(X, w, w2, w3, w4, p_drop_conv, p_drop_hidden):
 2    
 3    # X:  128 * 1 * 28 * 28
 4    # w:  32 * 1 * 3 * 3
 5    # full mode
 6    # l1a: 128 * 32 * (28 + 3 - 1) * (28 + 3 - 1)
 7    l1a = rectify(conv2d(X, w, border_mode='full'))
 8    # l1a: 128 * 32 * 30 * 30
 9    # ignore_border False
10    # l1:  128 * 32 * (30 / 2) * (30 / 2)
11    l1 = max_pool_2d(l1a, (2, 2), ignore_border=False)
12    l1 = dropout(l1, p_drop_conv)
13
14    # l1:  128 * 32 * 15 * 15
15    # w2:  64 * 32 * 3 * 3
16    # valid mode
17    # l2a: 128 * 64 * (15 - 3 + 1) * (15 - 3 + 1)
18    l2a = rectify(conv2d(l1, w2))    
19    # l2a: 128 * 64 * 13 * 13
20    # l2:  128 * 64 * (13 / 2 + 1) * (13 / 2 + 1)
21    l2 = max_pool_2d(l2a, (2, 2), ignore_border=False)
22    l2 = dropout(l2, p_drop_conv)
23
24    # l2:  128 * 64 * 7 * 7
25    # w3:  128 * 64 * 3 * 3
26    # l3a: 128 * 128 * (7 - 3 + 1) * (7 - 3 + 1)
27    l3a = rectify(conv2d(l2, w3))
28    # l3a: 128 * 128 * 5 * 5
29    # l3b: 128 * 128 * (5 / 2 + 1) * (5 / 2 + 1)
30    l3b = max_pool_2d(l3a, (2, 2), ignore_border=False)    
31    # l3b: 128 * 128 * 3 * 3
32    # l3:  128 * (128 * 3 * 3)
33    l3 = T.flatten(l3b, outdim=2)
34    l3 = dropout(l3, p_drop_conv)
35    
36    # l3: 128 * (128 * 3 * 3)
37    # w4: (128 * 3 * 3) * 625
38    # l4: 128 * 625
39    l4 = rectify(T.dot(l3, w4))
40    l4 = dropout(l4, p_drop_hidden)
41
42    # l5:  128 * 625
43    # w5:  625 * 10
44    # pyx: 128 * 10
45    pyx = softmax(T.dot(l4, w_o))
46    return l1, l2, l3, l4, pyx

1X = T.ftensor4()
2Y = T.fmatrix()
3
4w = init_weights((32, 1, 3, 3))
5w2 = init_weights((64, 32, 3, 3))
6w3 = init_weights((128, 64, 3, 3))
7w4 = init_weights((128 * 3 * 3, 625))
8w_o = init_weights((625, 10))

1noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, 0.2, 0.5)

1l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, 0., 0.)
2y_x = T.argmax(py_x, axis=1)

1cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))
2params = [w, w2, w3, w4, w_o]
3updates = RMSprop(cost, params, lr=0.001)

1train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)
2predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)
3
4for i in range(50):
5    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
6        cost = train(trX[start:end], trY[start:end])
7    print "iter {:03d}, {:.3f}".format(i + 1, np.mean(np.argmax(teY, axis=1) == predict(teX)))