Simple neural network train with deeplearning4j

Question

I am starting to use deeplearning4j and I am trying to create a simple neural network.

I want to bring the function closer sin(x)/x. This is theoretically possible with a single hidden layer.

First I create a simulated dataset (x,y), and then I try to approximate the function by a neural network with 20 hidden nodes and a sigmoid activation function. Unfortunately, a value evaluated using NN y_estdoes not even close the actual value y.

I wonder where the error is.

this is my current code:

package org.deeplearning4j.examples.myexamples

import org.deeplearning4j.nn.api.OptimizationAlgorithm
import org.deeplearning4j.nn.conf.{ MultiLayerConfiguration, NeuralNetConfiguration }
import org.deeplearning4j.nn.conf.layers.OutputLayer
import org.deeplearning4j.nn.conf.layers.DenseLayer
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork
import org.deeplearning4j.nn.weights.WeightInit
import org.deeplearning4j.optimize.listeners.ScoreIterationListener
import org.nd4j.linalg.api.ops.impl.transforms.Sin
import org.nd4j.linalg.dataset.DataSet
import org.nd4j.linalg.factory.Nd4j
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction

import org.nd4j.linalg.api.ndarray.INDArray
import scalax.chart.api._

import breeze.linalg.linspace

/**
 * Created by donbeo on 16/10/15.
 */
package object MyExample1 {
  def main(args: Array[String]) = {

    def plotXY(x:INDArray, y:INDArray):Unit = {

      val dataPlot = for(i <- 0 to y.length()-1) yield (x.getFloat(i), y.getFloat(i))
      val chart = XYLineChart(dataPlot)
      chart.show()
    }


    val nSamples = 500
    val xMin = -4
    val xMax = 4

    val x0 = linspace(xMin, xMax, nSamples)
    val y0 = breeze.numerics.sin(x0) / x0

    val x = Nd4j.create(x0.toArray).reshape(nSamples, 1)
    val y = Nd4j.create(y0.toArray).reshape(nSamples, 1)

    plotXY(x, y)


    val numInputs = 1
    val numOutputs = 1
    val numHiddenNodes = 20

    val seed = 123
    val iterations = 100


    val conf = new NeuralNetConfiguration.Builder()
      .seed(seed)
      .iterations(iterations)
      .optimizationAlgo(OptimizationAlgorithm.LBFGS)
      .list(2)
      .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes)
        .activation("sigmoid")
        .weightInit(WeightInit.XAVIER)
        .build())
      .layer(1, new OutputLayer.Builder(LossFunction.MSE)
        .weightInit(WeightInit.XAVIER)
        .activation("identity")
        .nIn(numHiddenNodes).nOut(numOutputs).build())
      .build()

    val dataSet = new DataSet(x, y)
    val network: MultiLayerNetwork = new MultiLayerNetwork(conf)
    network.init()
    network.setListeners(new ScoreIterationListener(1))
    network.fit(dataSet)

    val y_est = network.output(x)

    plotXY(x, y_est)

  }
}

Answer 1

. , .

package org.deeplearning4j.examples.deepbelief;

import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.transforms.Sin;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions;

/**
 * Created by agibsonccc on 10/17/15.
 */
public class RandomValues {
    public static void main(String[] args) {
        Nd4j.ENFORCE_NUMERICAL_STABILITY = true;
        int numInputs = 1;
        int numOutputs = 1;
        int numHiddenNodes = 20;
        int nSamples = 500;
        INDArray x0 = Nd4j.linspace(-10, 10, 500).reshape(nSamples,1);
        INDArray y0 = Nd4j.getExecutioner().execAndReturn(new Sin(x0, x0.dup())).div(x0);
        System.out.println(y0);

        int seed = 123;
        int iterations = 100;
        MultiLayerConfiguration conf  = new NeuralNetConfiguration.Builder()
                .seed(seed).constrainGradientToUnitNorm(true).learningRate(1e-1)
                .iterations(iterations).constrainGradientToUnitNorm(true).l1(1e-1)
                .l2(1e-3).regularization(true).miniBatch(false)
                .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT)
                .list(2)
                .layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes)
                        .activation("relu")
                        .weightInit(WeightInit.XAVIER)
                        .build())
                .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.RMSE_XENT)
                        .weightInit(WeightInit.XAVIER).updater(Updater.SGD)
                        .activation("identity").weightInit(WeightInit.XAVIER)
                        .nIn(numHiddenNodes).nOut(numOutputs).build()).backprop(true)
                .build();

        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();
        network.setListeners(new ScoreIterationListener(1));
        network.fit(new DataSet(x0, y0));
        System.out.println(network.output(x0));

    }


}