Download our e-book of Introduction To Python
Neha Kumawat
4 months ago
def adagrad():
weights, bais, eta = init_w, init_b, 0.1
v_w, v_b, eps = 0, 0, 1e-8
for i in range(max_epochs):
dw, db = 0, 0
for x,y in zip(X,Y):
dw += grad_w(weights, bais, x, y)
db += grad_b(weights, bais, x, y)
v_w = v_w + dw**2
v_b = v_b + db**2
weights_2 = weights - (eta / np.sqrt(v_w + eps)) * dw
bais_2 = bais - (eta / np.sqrt(v_b + eps)) * db
# Adadalta
def adadelta(params, sqrs, deltas, rho, batch_size):
eps_stable = 1e-5
for param, sqr, delta in zip(params, sqrs, deltas):
g = param.grad / batch_size
sqr[:] = rho * sqr + (1. - rho) * nd.square(g)
cur_delta = nd.sqrt(delta + eps_stable) / nd.sqrt(sqr + eps_stable) * g
delta[:] = rho * delta + (1. - rho) * cur_delta * cur_delta
# update weight
param[:] -= cur_delta