Search in sources :

Example 1 with ParameterServerNode

use of org.nd4j.parameterserver.node.ParameterServerNode in project deeplearning4j by deeplearning4j.

the class ParameterServerParallelWrapper method init.

private void init(Object iterator) {
    if (numEpochs < 1)
        throw new IllegalStateException("numEpochs must be >= 1");
    //TODO: make this efficient
    if (iterator instanceof DataSetIterator) {
        DataSetIterator dataSetIterator = (DataSetIterator) iterator;
        numUpdatesPerEpoch = numUpdatesPerEpoch(dataSetIterator);
    } else if (iterator instanceof MultiDataSetIterator) {
        MultiDataSetIterator iterator1 = (MultiDataSetIterator) iterator;
        numUpdatesPerEpoch = numUpdatesPerEpoch(iterator1);
    } else
        throw new IllegalArgumentException("Illegal type of object passed in for initialization. Must be of type DataSetIterator or MultiDataSetIterator");
    mediaDriverContext = new MediaDriver.Context();
    mediaDriver = MediaDriver.launchEmbedded(mediaDriverContext);
    parameterServerNode = new ParameterServerNode(mediaDriver, statusServerPort, numWorkers);
    running = new AtomicBoolean(true);
    if (parameterServerArgs == null)
        parameterServerArgs = new String[] { "-m", "true", "-s", "1," + String.valueOf(model.numParams()), "-p", "40323", "-h", "localhost", "-id", "11", "-md", mediaDriver.aeronDirectoryName(), "-sh", "localhost", "-sp", String.valueOf(statusServerPort), "-u", String.valueOf(numUpdatesPerEpoch) };
    if (numWorkers == 0)
        numWorkers = Runtime.getRuntime().availableProcessors();
    linkedBlockingQueue = new LinkedBlockingQueue<>(numWorkers);
    //pass through args for the parameter server subscriber
    parameterServerNode.runMain(parameterServerArgs);
    while (!parameterServerNode.subscriberLaunched()) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    try {
        Thread.sleep(10000);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    log.info("Parameter server started");
    parameterServerClient = new Trainer[numWorkers];
    executorService = Executors.newFixedThreadPool(numWorkers);
    for (int i = 0; i < numWorkers; i++) {
        Model model = null;
        if (this.model instanceof ComputationGraph) {
            ComputationGraph computationGraph = (ComputationGraph) this.model;
            model = computationGraph.clone();
        } else if (this.model instanceof MultiLayerNetwork) {
            MultiLayerNetwork multiLayerNetwork = (MultiLayerNetwork) this.model;
            model = multiLayerNetwork.clone();
        }
        parameterServerClient[i] = new Trainer(ParameterServerClient.builder().aeron(parameterServerNode.getAeron()).ndarrayRetrieveUrl(parameterServerNode.getSubscriber()[i].getResponder().connectionUrl()).ndarraySendUrl(parameterServerNode.getSubscriber()[i].getSubscriber().connectionUrl()).subscriberHost("localhost").masterStatusHost("localhost").masterStatusPort(statusServerPort).subscriberPort(40625 + i).subscriberStream(12 + i).build(), running, linkedBlockingQueue, model);
        final int j = i;
        executorService.submit(() -> parameterServerClient[j].start());
    }
    init = true;
    log.info("Initialized wrapper");
}
Also used : ParameterServerNode(org.nd4j.parameterserver.node.ParameterServerNode) MultiDataSetIterator(org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator) AsyncMultiDataSetIterator(org.deeplearning4j.datasets.iterator.AsyncMultiDataSetIterator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MediaDriver(io.aeron.driver.MediaDriver) Model(org.deeplearning4j.nn.api.Model) ComputationGraph(org.deeplearning4j.nn.graph.ComputationGraph) MultiLayerNetwork(org.deeplearning4j.nn.multilayer.MultiLayerNetwork) DataSetIterator(org.nd4j.linalg.dataset.api.iterator.DataSetIterator) AsyncDataSetIterator(org.deeplearning4j.datasets.iterator.AsyncDataSetIterator) MultiDataSetIterator(org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator) AsyncMultiDataSetIterator(org.deeplearning4j.datasets.iterator.AsyncMultiDataSetIterator)

Aggregations

MediaDriver (io.aeron.driver.MediaDriver)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AsyncDataSetIterator (org.deeplearning4j.datasets.iterator.AsyncDataSetIterator)1 AsyncMultiDataSetIterator (org.deeplearning4j.datasets.iterator.AsyncMultiDataSetIterator)1 Model (org.deeplearning4j.nn.api.Model)1 ComputationGraph (org.deeplearning4j.nn.graph.ComputationGraph)1 MultiLayerNetwork (org.deeplearning4j.nn.multilayer.MultiLayerNetwork)1 DataSetIterator (org.nd4j.linalg.dataset.api.iterator.DataSetIterator)1 MultiDataSetIterator (org.nd4j.linalg.dataset.api.iterator.MultiDataSetIterator)1 ParameterServerNode (org.nd4j.parameterserver.node.ParameterServerNode)1