All geek questions in one place

Keras - a problem with the implementation of the Rprop algorithm

I am trying to implement an optimized backpropagation optimizer for Keras ( link ), but the hard part was able to update each individual parameter based on whether its corresponding gradient is positive, negative or zero. I wrote the code below as the start of the Rprop optimizer implementation. However, I cannot find a way to access the parameters individually. Looping over params(as in the code below) returns p, g, g_old, s, wChangeOldat each iteration, which are all matrices.

Is there a way in which I could iterate over individual parameters and update them? It would also work if I could index the parameter vector based on the sign of its gradients. Thank!

class Rprop(Optimizer):
    def __init__(self, init_step=0.01, **kwargs):
        super(Rprop, self).__init__(**kwargs)
        self.init_step = K.variable(init_step, name='init_step')
        self.iterations = K.variable(0., name='iterations')

        self.posStep = 1.2
        self.negStep = 0.5
        self.minStep = 1e-6
        self.maxStep = 50.

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        shapes = [K.get_variable_shape(p) for p in params]
        stepList = [K.ones(shape)*self.init_step  for shape in shapes]
        wChangeOldList = [K.zeros(shape) for shape in shapes]
        grads_old = [K.zeros(shape) for shape in shapes]

        self.weights = stepList + grads_old + wChangeOldList
        self.updates = []

        for p, g, g_old, s, wChangeOld in zip(params, grads, grads_old, 
                                                                  stepList, wChangeOldList):
            change = K.sign(g * g_old)

            if change > 0:
                s_new = K.minimum(s * self.posStep, self.maxStep)
                wChange = s_new * K.sign(g)
                g_new = g

            elif change < 0:
                s_new = K.maximum(s * self.posStep, self.maxStep)
                wChange = - wChangeOld
                g_new = 0

            else:
                s_new = s
                wChange = s_new * K.sign(g)
                g_new = p

            self.updates.append(K.update(g_old, g_new))
            self.updates.append(K.update(wChangeOld, wChange))
            self.updates.append(K.update(s, s_new))

            new_p = p - wChange

            # Apply constraints
            if p in constraints:
                c = constraints[p]
                new_p = c(new_p)

            self.updates.append(K.update(p, new_p))
        return self.updates

    def get_config(self):
        config = {'init_step': float(K.get_value(self.init_step))}
        base_config = super(Rprop, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))
+4
2

RProp Keras . . , .

: keras, theano ( ). anano backend, .

class RProp(Optimizer):
    def __init__(self, init_alpha=1e-3, scale_up=1.2, scale_down=0.5, min_alpha=1e-6, max_alpha=50., **kwargs):
        super(RProp, self).__init__(**kwargs)
        self.init_alpha = K.variable(init_alpha, name='init_alpha')
        self.scale_up = K.variable(scale_up, name='scale_up')
        self.scale_down = K.variable(scale_down, name='scale_down')
        self.min_alpha = K.variable(min_alpha, name='min_alpha')
        self.max_alpha = K.variable(max_alpha, name='max_alpha')

    def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [K.get_variable_shape(p) for p in params]
        alphas = [K.variable(numpy.ones(shape) * self.init_alpha) for shape in shapes]
        old_grads = [K.zeros(shape) for shape in shapes]
        self.weights = alphas + old_grads
        self.updates = []

        for param, grad, old_grad, alpha in zip(params, grads, old_grads, alphas):
            new_alpha = K.switch(
                K.greater(grad * old_grad, 0),
                K.minimum(alpha * self.scale_up, self.max_alpha),
                K.maximum(alpha * self.scale_down, self.min_alpha)
            )
            new_param = param - K.sign(grad) * new_alpha
            # Apply constraints
            if param in constraints:
                c = constraints[param]
                new_param = c(new_param)
            self.updates.append(K.update(param, new_param))
            self.updates.append(K.update(alpha, new_alpha))
            self.updates.append(K.update(old_grad, grad))

        return self.updates

    def get_config(self):
        config = {
            'init_alpha': float(K.get_value(self.init_alpha)),
            'scale_up': float(K.get_value(self.scale_up)),
            'scale_down': float(K.get_value(self.scale_down)),
            'min_alpha': float(K.get_value(self.min_alpha)),
            'max_alpha': float(K.get_value(self.max_alpha)),
        }
        base_config = super(RProp, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

:

  • RProp - : , . , .
  • (Keras builtin) RProp. , , , , , , :)

( ):

  • wChange , .
  • change > 0 , , change . , , K.switch().
  • maxStep minStep .
  • , change , , .
  • g_new = 0 g_new = p g_new = g if.
0

keras Pyhton, . - . backpropagation, . Tensorflow CNTK .

Rprop : https://pdfs.semanticscholar.org/df9c/6a3843d54a28138a596acc85a96367a064c2.pdf

class iRprop_(Optimizer):
def __init__(self, init_alpha=0.01, scale_up=1.2, scale_down=0.5, min_alpha=0.00001, max_alpha=50., **kwargs):
    super(iRprop_, self).__init__(**kwargs)
    self.init_alpha = K.variable(init_alpha, name='init_alpha')
    self.scale_up = K.variable(scale_up, name='scale_up')
    self.scale_down = K.variable(scale_down, name='scale_down')
    self.min_alpha = K.variable(min_alpha, name='min_alpha')
    self.max_alpha = K.variable(max_alpha, name='max_alpha')

def get_updates(self, params, loss):
    grads = self.get_gradients(loss, params)
    shapes = [K.get_variable_shape(p) for p in params]
    alphas = [K.variable(K.ones(shape) * self.init_alpha) for shape in shapes]
    old_grads = [K.zeros(shape) for shape in shapes]
    self.weights = alphas + old_grads
    self.updates = []

    for p, grad, old_grad, alpha in zip(params, grads, old_grads, alphas):
        grad = K.sign(grad)
        new_alpha = K.switch(
            K.greater(grad * old_grad, 0),
            K.minimum(alpha * self.scale_up, self.max_alpha),
            K.switch(K.less(grad * old_grad, 0),K.maximum(alpha * self.scale_down, self.min_alpha),alpha)    
        )

        grad = K.switch(K.less(grad * old_grad, 0),K.zeros_like(grad),grad)
        new_p = p - grad * new_alpha 

        # Apply constraints.
        if getattr(p, 'constraint', None) is not None:
            new_p = p.constraint(new_p)
        self.updates.append(K.update(p, new_p))
        self.updates.append(K.update(alpha, new_alpha))
        self.updates.append(K.update(old_grad, grad))

    return self.updates

def get_config(self):
    config = {
        'init_alpha': float(K.get_value(self.init_alpha)),
        'scale_up': float(K.get_value(self.scale_up)),
        'scale_down': float(K.get_value(self.scale_down)),
        'min_alpha': float(K.get_value(self.min_alpha)),
        'max_alpha': float(K.get_value(self.max_alpha)),
    }
    base_config = super(iRprop_, self).get_config()
    return dict(list(base_config.items()) + list(config.items()))
0

Source: https://habr.com/ru/post/1676296/

More articles:


All Articles