Examples with VecsInput hex.Layer.VecsInput used on opensource projects

Example 1 with VecsInput

use of hex.Layer.VecsInput in project h2o-2 by h2oai.

the class NeuralNetMnistPretrain method preTrain.

private final void preTrain(Layer[] ls, int index, int epochs) {
    // Build a network with same layers below 'index', and an auto-encoder at the top
    Layer[] pre = new Layer[index + 2];
    VecsInput input = (VecsInput) ls[0];
    pre[0] = new VecsInput(input.vecs, input);
    //clone the parameters
    pre[0].init(pre, 0, ls[0].params);
    for (int i = 1; i < index; i++) {
        //pre[i] = new Layer.Rectifier(ls[i].units);
        pre[i] = new Layer.Tanh(ls[i].units);
        Layer.shareWeights(ls[i], pre[i]);
        //share the parameters
        pre[i].init(pre, i, ls[i].params);
        //turn off training for these layers
        pre[i].params.rate = 0;
    }
    // Auto-encoder is a layer and a reverse layer on top
    //pre[index] = new Layer.Rectifier(ls[index].units);
    //pre[index + 1] = new Layer.RectifierPrime(ls[index - 1].units);
    pre[index] = new Layer.Tanh(ls[index].units);
    pre[index].init(pre, index, ls[index].params);
    pre[index].params.rate = 1e-5;
    pre[index + 1] = new Layer.TanhPrime(ls[index - 1].units);
    pre[index + 1].init(pre, index + 1, pre[index].params);
    pre[index + 1].params.rate = 1e-5;
    Layer.shareWeights(ls[index], pre[index]);
    Layer.shareWeights(ls[index], pre[index + 1]);
    _trainer = new Trainer.Direct(pre, epochs, self());
    // Basic visualization of images and weights
    JFrame frame = new JFrame("H2O Pre-Training");
    frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
    MnistCanvas canvas = new MnistCanvas(_trainer);
    frame.setContentPane(canvas.init());
    frame.pack();
    frame.setLocationRelativeTo(null);
    frame.setVisible(true);
    _trainer.start();
    _trainer.join();
}

Example 2 with VecsInput

use of hex.Layer.VecsInput in project h2o-2 by h2oai.

the class NeuralNetMnistPretrain method build.

@Override
protected Layer[] build(Vec[] data, Vec labels, VecsInput inputStats, VecSoftmax outputStats) {
    Layer[] ls = new Layer[4];
    ls[0] = new VecsInput(data, inputStats);
    //    ls[1] = new Layer.RectifierDropout(1024);
    //    ls[2] = new Layer.RectifierDropout(1024);
    ls[1] = new Layer.Tanh(50);
    ls[2] = new Layer.Tanh(50);
    ls[3] = new VecSoftmax(labels, outputStats);
    // Parameters for MNIST run
    NeuralNet p = new NeuralNet();
    //only used for NN run after pretraining
    p.rate = 0.01;
    p.activation = NeuralNet.Activation.Tanh;
    p.loss = NeuralNet.Loss.CrossEntropy;
    //    p.rate_annealing = 1e-6f;
    //    p.max_w2 = 15;
    //    p.momentum_start = 0.5f;
    //    p.momentum_ramp = 60000 * 300;
    //    p.momentum_stable = 0.99f;
    //    p.l1 = .00001f;
    //    p.l2 = .00f;
    p.initial_weight_distribution = NeuralNet.InitialWeightDistribution.UniformAdaptive;
    for (int i = 0; i < ls.length; i++) {
        ls[i].init(ls, i, p);
    }
    return ls;
}

Example 3 with VecsInput

use of hex.Layer.VecsInput in project h2o-2 by h2oai.

the class MnistCanvas method paint.

@Override
public void paint(Graphics g) {
    Layer[] ls = _trainer.layers();
    Vec[] vecs = ((VecsInput) ls[0]).vecs;
    //    Vec resp = ((VecSoftmax) ls[ls.length - 1]).vec;
    int edge = 56, pad = 10;
    int rand = _rand.nextInt((int) vecs[0].length());
    // Side
    {
        BufferedImage in = new BufferedImage(EDGE, EDGE, BufferedImage.TYPE_INT_RGB);
        WritableRaster r = in.getRaster();
        // Input
        int[] pix = new int[PIXELS];
        for (int i = 0; i < pix.length; i++) pix[i] = (int) (vecs[i].at8(rand));
        r.setDataElements(0, 0, EDGE, EDGE, pix);
        g.drawImage(in, pad, pad, null);
    // Labels
    //      g.drawString("" + resp.at8(rand), 10, 50);
    //      g.drawString("RBM " + _level, 10, 70);
    }
    // Outputs
    int offset = pad;
    //    float[] visible = new float[MnistNeuralNetTest.PIXELS];
    //    System.arraycopy(_images, rand * MnistNeuralNetTest.PIXELS, visible, 0, MnistNeuralNetTest.PIXELS);
    //    for( int i = 0; i <= _level; i++ ) {
    //      for( int pass = 0; pass < 10; pass++ ) {
    //        if( i == _level ) {
    //          int[] output = new int[visible.length];
    //          for( int v = 0; v < visible.length; v++ )
    //            output[v] = (int) Math.min(visible[v] * 255, 255);
    //          BufferedImage out = new BufferedImage(MnistNeuralNetTest.EDGE, MnistNeuralNetTest.EDGE,
    //              BufferedImage.TYPE_INT_RGB);
    //          WritableRaster r = out.getRaster();
    //          r.setDataElements(0, 0, MnistNeuralNetTest.EDGE, MnistNeuralNetTest.EDGE, output);
    //          BufferedImage image = new BufferedImage(edge, edge, BufferedImage.TYPE_INT_RGB);
    //          Graphics2D ig = image.createGraphics();
    //          ig.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
    //          ig.clearRect(0, 0, edge, edge);
    //          ig.drawImage(out, 0, 0, edge, edge, null);
    //          ig.dispose();
    //          g.drawImage(image, pad * 2 + MnistNeuralNetTest.EDGE, offset, null);
    //          offset += pad + edge;
    //        }
    //        if( _ls[i]._v != null ) {
    //          float[] hidden = new float[_ls[i]._b.length];
    //          _ls[i].forward(visible, hidden);
    //          visible = _ls[i].generate(hidden);
    //        }
    //      }
    //      float[] t = new float[_ls[i]._b.length];
    //      _ls[i].forward(visible, t);
    //      visible = t;
    //    }
    // Weights
    int buf = EDGE + pad + pad;
    Layer layer = ls[_level];
    double mean = 0;
    int n = layer._w.length;
    for (int i = 0; i < n; i++) mean += layer._w[i];
    mean /= layer._w.length;
    double sigma = 0;
    for (int i = 0; i < layer._w.length; i++) {
        double d = layer._w[i] - mean;
        sigma += d * d;
    }
    sigma = Math.sqrt(sigma / (layer._w.length - 1));
    for (int o = 0; o < layer._b.length; o++) {
        if (o % 10 == 0) {
            offset = pad;
            buf += pad + edge;
        }
        int[] start = new int[layer._previous._a.length];
        for (int i = 0; i < layer._previous._a.length; i++) {
            double w = layer._w[o * layer._previous._a.length + i];
            w = ((w - mean) / sigma) * 200;
            if (w >= 0)
                //GREEN
                start[i] = ((int) Math.min(+w, 255)) << 8;
            else
                //RED
                start[i] = ((int) Math.min(-w, 255)) << 16;
        }
        BufferedImage out = new BufferedImage(EDGE, EDGE, BufferedImage.TYPE_INT_RGB);
        WritableRaster r = out.getRaster();
        r.setDataElements(0, 0, EDGE, EDGE, start);
        BufferedImage resized = new BufferedImage(edge, edge, BufferedImage.TYPE_INT_RGB);
        Graphics2D g2 = resized.createGraphics();
        try {
            g2.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BICUBIC);
            g2.clearRect(0, 0, edge, edge);
            g2.drawImage(out, 0, 0, edge, edge, null);
        } finally {
            g2.dispose();
        }
        g.drawImage(resized, buf, offset, null);
        offset += pad + edge;
    }
}

Example 4 with VecsInput

use of hex.Layer.VecsInput in project h2o-2 by h2oai.

the class NeuralNetMnist method build.

protected Layer[] build(Vec[] data, Vec labels, VecsInput inputStats, VecSoftmax outputStats) {
    //same parameters as in test_NN_mnist.py
    Layer[] ls = new Layer[5];
    ls[0] = new VecsInput(data, inputStats);
    ls[1] = new Layer.RectifierDropout(117);
    ls[2] = new Layer.RectifierDropout(131);
    ls[3] = new Layer.RectifierDropout(129);
    ls[ls.length - 1] = new VecSoftmax(labels, outputStats);
    NeuralNet p = new NeuralNet();
    p.seed = 98037452452l;
    p.rate = 0.005;
    p.rate_annealing = 1e-6;
    p.activation = NeuralNet.Activation.RectifierWithDropout;
    p.loss = NeuralNet.Loss.CrossEntropy;
    p.input_dropout_ratio = 0.2;
    p.max_w2 = 15;
    p.epochs = 2;
    p.l1 = 1e-5;
    p.l2 = 0.0000001;
    p.momentum_start = 0.5;
    p.momentum_ramp = 100000;
    p.momentum_stable = 0.99;
    p.initial_weight_distribution = NeuralNet.InitialWeightDistribution.UniformAdaptive;
    p.classification = true;
    p.diagnostics = true;
    p.expert_mode = true;
    for (int i = 0; i < ls.length; i++) {
        ls[i].init(ls, i, p);
    }
    return ls;
}

Example 5 with VecsInput

use of hex.Layer.VecsInput in project h2o-2 by h2oai.

the class NeuralNetMnist method execImpl.

@Override
protected void execImpl() {
    Frame trainf = TestUtil.parseFromH2OFolder("smalldata/mnist/train.csv.gz");
    Frame testf = TestUtil.parseFromH2OFolder("smalldata/mnist/test.csv.gz");
    train = trainf.vecs();
    test = testf.vecs();
    // Labels are on last column for this dataset
    final Vec trainLabels = train[train.length - 1];
    train = Utils.remove(train, train.length - 1);
    final Vec testLabels = test[test.length - 1];
    test = Utils.remove(test, test.length - 1);
    final Layer[] ls = build(train, trainLabels, null, null);
    // Monitor training
    final Timer timer = new Timer();
    final long start = System.nanoTime();
    final AtomicInteger evals = new AtomicInteger(1);
    timer.schedule(new TimerTask() {

        @Override
        public void run() {
            if (!Job.isRunning(self()))
                timer.cancel();
            else {
                double time = (System.nanoTime() - start) / 1e9;
                Trainer trainer = _trainer;
                long processed = trainer == null ? 0 : trainer.processed();
                int ps = (int) (processed / time);
                String text = (int) time + "s, " + processed + " samples (" + (ps) + "/s) ";
                // Build separate nets for scoring purposes, use same normalization stats as for training
                Layer[] temp = build(train, trainLabels, (VecsInput) ls[0], (VecSoftmax) ls[ls.length - 1]);
                Layer.shareWeights(ls, temp);
                // Estimate training error on subset of dataset for speed
                Errors e = NeuralNet.eval(temp, 1000, null);
                text += "train: " + e;
                text += ", rate: ";
                text += String.format("%.5g", ls[0].rate(processed));
                text += ", momentum: ";
                text += String.format("%.5g", ls[0].momentum(processed));
                System.out.println(text);
                if ((evals.incrementAndGet() % 1) == 0) {
                    System.out.println("Computing test error");
                    temp = build(test, testLabels, (VecsInput) ls[0], (VecSoftmax) ls[ls.length - 1]);
                    Layer.shareWeights(ls, temp);
                    e = NeuralNet.eval(temp, 0, null);
                    System.out.println("Test error: " + e);
                }
            }
        }
    }, 0, 10);
    startTraining(ls);
}