Examples with DotAggregation org.nd4j.parameterserver.distributed.messages.aggregations.DotAggregation used on opensource projects

Example 1 with DotAggregation

use of org.nd4j.parameterserver.distributed.messages.aggregations.DotAggregation in project nd4j by deeplearning4j.

the class VoidParameterServerTest method testNodeInitialization2.

/**
 * This is very important test, it covers basic messages handling over network.
 * Here we have 1 client, 1 connected Shard + 2 shards available over multicast UDP
 *
 * PLEASE NOTE: This test uses manual stepping through messages
 *
 * @throws Exception
 */
@Test
public void testNodeInitialization2() throws Exception {
    final AtomicInteger failCnt = new AtomicInteger(0);
    final AtomicInteger passCnt = new AtomicInteger(0);
    final AtomicInteger startCnt = new AtomicInteger(0);
    INDArray exp = Nd4j.create(new double[] { 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00 });
    final VoidConfiguration clientConf = VoidConfiguration.builder().unicastPort(34567).multicastPort(45678).numberOfShards(3).shardAddresses(localIPs).multicastNetwork("224.0.1.1").streamId(119).forcedRole(NodeRole.CLIENT).ttl(4).build();
    final VoidConfiguration shardConf1 = VoidConfiguration.builder().unicastPort(34567).multicastPort(45678).numberOfShards(3).streamId(119).shardAddresses(localIPs).multicastNetwork("224.0.1.1").ttl(4).build();
    final VoidConfiguration shardConf2 = // we'll never get anything on this port
    VoidConfiguration.builder().unicastPort(34569).multicastPort(45678).numberOfShards(3).streamId(119).shardAddresses(localIPs).multicastNetwork("224.0.1.1").ttl(4).build();
    final VoidConfiguration shardConf3 = // we'll never get anything on this port
    VoidConfiguration.builder().unicastPort(34570).multicastPort(45678).numberOfShards(3).streamId(119).shardAddresses(localIPs).multicastNetwork("224.0.1.1").ttl(4).build();
    VoidParameterServer clientNode = new VoidParameterServer(true);
    clientNode.setShardIndex((short) 0);
    clientNode.init(clientConf);
    clientNode.getTransport().launch(Transport.ThreadingModel.DEDICATED_THREADS);
    assertEquals(NodeRole.CLIENT, clientNode.getNodeRole());
    Thread[] threads = new Thread[3];
    final VoidConfiguration[] voidConfigurations = new VoidConfiguration[] { shardConf1, shardConf2, shardConf3 };
    VoidParameterServer[] shards = new VoidParameterServer[threads.length];
    for (int t = 0; t < threads.length; t++) {
        final int x = t;
        threads[t] = new Thread(() -> {
            shards[x] = new VoidParameterServer(true);
            shards[x].setShardIndex((short) x);
            shards[x].init(voidConfigurations[x]);
            shards[x].getTransport().launch(Transport.ThreadingModel.DEDICATED_THREADS);
            assertEquals(NodeRole.SHARD, shards[x].getNodeRole());
            startCnt.incrementAndGet();
            passCnt.incrementAndGet();
        });
        threads[t].setDaemon(true);
        threads[t].start();
    }
    // we block until all threads are really started before sending commands
    while (startCnt.get() < threads.length) Thread.sleep(500);
    // give additional time to start handlers
    Thread.sleep(1000);
    // now we'll send commands from Client, and we'll check how these messages will be handled
    DistributedInitializationMessage message = DistributedInitializationMessage.builder().numWords(100).columnsPerShard(10).seed(123).useHs(false).useNeg(true).vectorLength(100).build();
    log.info("MessageType: {}", message.getMessageType());
    clientNode.getTransport().sendMessage(message);
    // now we check message queue within Shards
    for (int t = 0; t < threads.length; t++) {
        VoidMessage incMessage = shards[t].getTransport().takeMessage();
        assertNotEquals("Failed for shard " + t, null, incMessage);
        assertEquals("Failed for shard " + t, message.getMessageType(), incMessage.getMessageType());
        // we should put message back to corresponding
        shards[t].getTransport().putMessage(incMessage);
    }
    for (int t = 0; t < threads.length; t++) {
        VoidMessage incMessage = shards[t].getTransport().takeMessage();
        assertNotEquals("Failed for shard " + t, null, incMessage);
        shards[t].handleMessage(message);
        /**
         * Now we're checking how data storage was initialized
         */
        assertEquals(null, shards[t].getNegTable());
        assertEquals(null, shards[t].getSyn1());
        assertNotEquals(null, shards[t].getExpTable());
        assertNotEquals(null, shards[t].getSyn0());
        assertNotEquals(null, shards[t].getSyn1Neg());
    }
    // now we'll check passing for negTable, but please note - we're not sending it right now
    INDArray negTable = Nd4j.create(100000).assign(12.0f);
    DistributedSolidMessage negMessage = new DistributedSolidMessage(WordVectorStorage.NEGATIVE_TABLE, negTable, false);
    for (int t = 0; t < threads.length; t++) {
        shards[t].handleMessage(negMessage);
        assertNotEquals(null, shards[t].getNegTable());
        assertEquals(negTable, shards[t].getNegTable());
    }
    // now we assign each row to something
    for (int t = 0; t < threads.length; t++) {
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_0, 1, (double) t));
        assertEquals(Nd4j.create(message.getColumnsPerShard()).assign((double) t), shards[t].getSyn0().getRow(1));
    }
    // and now we'll request for aggregated vector for row 1
    clientNode.getVector(1);
    VoidMessage vecm = shards[0].getTransport().takeMessage();
    assertEquals(7, vecm.getMessageType());
    VectorRequestMessage vrm = (VectorRequestMessage) vecm;
    assertEquals(1, vrm.getRowIndex());
    shards[0].handleMessage(vecm);
    Thread.sleep(100);
    // at this moment all 3 shards should already have distributed message
    for (int t = 0; t < threads.length; t++) {
        VoidMessage dm = shards[t].getTransport().takeMessage();
        assertEquals(20, dm.getMessageType());
        shards[t].handleMessage(dm);
    }
    // at this moment we should have messages propagated across all shards
    Thread.sleep(100);
    for (int t = threads.length - 1; t >= 0; t--) {
        VoidMessage msg;
        while ((msg = shards[t].getTransport().takeMessage()) != null) {
            shards[t].handleMessage(msg);
        }
    }
    // and at this moment, Shard_0 should contain aggregated vector for us
    assertEquals(true, shards[0].clipboard.isTracking(0L, 1L));
    assertEquals(true, shards[0].clipboard.isReady(0L, 1L));
    INDArray jointVector = shards[0].clipboard.nextCandidate().getAccumulatedResult();
    log.info("Joint vector: {}", jointVector);
    assertEquals(exp, jointVector);
    // first, we're setting data to something predefined
    for (int t = 0; t < threads.length; t++) {
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_0, 0, 0.0));
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_0, 1, 1.0));
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_0, 2, 2.0));
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_1_NEGATIVE, 0, 0.0));
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_1_NEGATIVE, 1, 1.0));
        shards[t].handleMessage(new DistributedAssignMessage(WordVectorStorage.SYN_1_NEGATIVE, 2, 2.0));
    }
    DistributedSgDotMessage ddot = new DistributedSgDotMessage(2L, new int[] { 0, 1, 2 }, new int[] { 0, 1, 2 }, 0, 1, new byte[] { 0, 1 }, true, (short) 0, 0.01f);
    for (int t = 0; t < threads.length; t++) {
        shards[t].handleMessage(ddot);
    }
    Thread.sleep(100);
    for (int t = threads.length - 1; t >= 0; t--) {
        VoidMessage msg;
        while ((msg = shards[t].getTransport().takeMessage()) != null) {
            shards[t].handleMessage(msg);
        }
    }
    // at this moment ot should be caclulated everywhere
    exp = Nd4j.create(new double[] { 0.0, 30.0, 120.0 });
    for (int t = 0; t < threads.length; t++) {
        assertEquals(true, shards[t].clipboard.isReady(0L, 2L));
        DotAggregation dot = (DotAggregation) shards[t].clipboard.unpin(0L, 2L);
        INDArray aggregated = dot.getAccumulatedResult();
        assertEquals(exp, aggregated);
    }
    for (int t = 0; t < threads.length; t++) {
        threads[t].join();
    }
    for (int t = 0; t < threads.length; t++) {
        shards[t].shutdown();
    }
    assertEquals(threads.length, passCnt.get());
    for (VoidParameterServer server : shards) {
        server.shutdown();
    }
    clientNode.shutdown();
}

Example 2 with DotAggregation

use of org.nd4j.parameterserver.distributed.messages.aggregations.DotAggregation in project nd4j by deeplearning4j.

the class DistributedSgDotMessage method processMessage.

/**
 * This method calculates dot of gives rows
 */
@Override
public void processMessage() {
    // this only picks up new training round
    // log.info("sI_{} Processing DistributedSgDotMessage taskId: {}", transport.getShardIndex(), getTaskId());
    SkipGramRequestMessage sgrm = new SkipGramRequestMessage(w1, w2, rowsB, codes, negSamples, alpha, 119);
    if (negSamples > 0) {
        // unfortunately we have to get copy of negSamples here
        int[] negatives = Arrays.copyOfRange(rowsB, codes.length, rowsB.length);
        sgrm.setNegatives(negatives);
    }
    sgrm.setTaskId(this.taskId);
    sgrm.setOriginatorId(this.getOriginatorId());
    // FIXME: get rid of THAT
    SkipGramTrainer sgt = (SkipGramTrainer) trainer;
    sgt.pickTraining(sgrm);
    // TODO: make this thing a single op, even specialOp is ok
    // we calculate dot for all involved rows
    int resultLength = codes.length + (negSamples > 0 ? (negSamples + 1) : 0);
    INDArray result = Nd4j.createUninitialized(resultLength, 1);
    int e = 0;
    for (; e < codes.length; e++) {
        double dot = Nd4j.getBlasWrapper().dot(storage.getArray(WordVectorStorage.SYN_0).getRow(w2), storage.getArray(WordVectorStorage.SYN_1).getRow(rowsB[e]));
        result.putScalar(e, dot);
    }
    // negSampling round
    for (; e < resultLength; e++) {
        double dot = Nd4j.getBlasWrapper().dot(storage.getArray(WordVectorStorage.SYN_0).getRow(w2), storage.getArray(WordVectorStorage.SYN_1_NEGATIVE).getRow(rowsB[e]));
        result.putScalar(e, dot);
    }
    if (voidConfiguration.getExecutionMode() == ExecutionMode.AVERAGING) {
        // just local bypass
        DotAggregation dot = new DotAggregation(taskId, (short) 1, shardIndex, result);
        dot.setTargetId((short) -1);
        dot.setOriginatorId(getOriginatorId());
        transport.putMessage(dot);
    } else if (voidConfiguration.getExecutionMode() == ExecutionMode.SHARDED) {
        // send this message to everyone
        DotAggregation dot = new DotAggregation(taskId, (short) voidConfiguration.getNumberOfShards(), shardIndex, result);
        dot.setTargetId((short) -1);
        dot.setOriginatorId(getOriginatorId());
        transport.sendMessage(dot);
    }
}

Example 3 with DotAggregation

use of org.nd4j.parameterserver.distributed.messages.aggregations.DotAggregation in project nd4j by deeplearning4j.

the class DistributedCbowDotMessage method processMessage.

/**
 * This method calculates dot of gives rows, with averaging applied to rowsA, as required by CBoW
 */
@Override
public void processMessage() {
    // this only picks up new training round
    // log.info("sI_{} Starting CBOW dot...", transport.getShardIndex());
    CbowRequestMessage cbrm = new CbowRequestMessage(rowsA, rowsB, w1, codes, negSamples, alpha, 119);
    if (negSamples > 0) {
        // unfortunately we have to get copy of negSamples here
        int[] negatives = Arrays.copyOfRange(rowsB, codes.length, rowsB.length);
        cbrm.setNegatives(negatives);
    }
    cbrm.setFrameId(-119L);
    cbrm.setTaskId(this.taskId);
    cbrm.setOriginatorId(this.getOriginatorId());
    // FIXME: get rid of THAT
    CbowTrainer cbt = (CbowTrainer) trainer;
    cbt.pickTraining(cbrm);
    // we calculate dot for all involved rows, and first of all we get mean word
    INDArray words = Nd4j.pullRows(storage.getArray(WordVectorStorage.SYN_0), 1, rowsA, 'c');
    INDArray mean = words.mean(0);
    int resultLength = codes.length + (negSamples > 0 ? (negSamples + 1) : 0);
    INDArray result = Nd4j.createUninitialized(resultLength, 1);
    int e = 0;
    for (; e < codes.length; e++) {
        double dot = Nd4j.getBlasWrapper().dot(mean, storage.getArray(WordVectorStorage.SYN_1).getRow(rowsB[e]));
        result.putScalar(e, dot);
    }
    // negSampling round
    for (; e < resultLength; e++) {
        double dot = Nd4j.getBlasWrapper().dot(mean, storage.getArray(WordVectorStorage.SYN_1_NEGATIVE).getRow(rowsB[e]));
        result.putScalar(e, dot);
    }
    if (voidConfiguration.getExecutionMode() == ExecutionMode.AVERAGING) {
        DotAggregation dot = new DotAggregation(taskId, (short) 1, shardIndex, result);
        dot.setTargetId((short) -1);
        dot.setOriginatorId(getOriginatorId());
        transport.putMessage(dot);
    } else if (voidConfiguration.getExecutionMode() == ExecutionMode.SHARDED) {
        // send this message to everyone
        DotAggregation dot = new DotAggregation(taskId, (short) voidConfiguration.getNumberOfShards(), shardIndex, result);
        dot.setTargetId((short) -1);
        dot.setOriginatorId(getOriginatorId());
        transport.sendMessage(dot);
    }
}