Examples with MediaDriver io.aeron.driver.MediaDriver used on opensource projects

Example 31 with MediaDriver

use of io.aeron.driver.MediaDriver in project nd4j by deeplearning4j.

the class ParameterServerSubscriber method run.

/**
 * @param args
 */
public void run(String[] args) {
    JCommander jcmdr = new JCommander(this);
    try {
        jcmdr.parse(args);
    } catch (ParameterException e) {
        e.printStackTrace();
        // User provides invalid input -> print the usage info
        jcmdr.usage();
        try {
            Thread.sleep(500);
        } catch (Exception e2) {
        }
        System.exit(1);
    }
    // ensure that the update opType is configured from the command line args
    updateType = UpdateType.valueOf(updateTypeString.toUpperCase());
    if (publishMasterUrl == null && !master)
        throw new IllegalStateException("Please specify a master url or set master to true");
    // for a remote one
    if (mediaDriver == null && mediaDriverDirectoryName == null) {
        // length of array * sizeof(float)
        int ipcLength = ArrayUtil.prod(Ints.toArray(shape)) * 4;
        // must be a power of 2
        ipcLength *= 2;
        // padding for NDArrayMessage
        ipcLength += 64;
        // Length in bytes for the SO_RCVBUF, 0 means use OS default. This needs to be larger than Receiver Window.
        System.setProperty("aeron.socket.so_rcvbuf", String.valueOf(ipcLength));
        final MediaDriver.Context mediaDriverCtx = new MediaDriver.Context().threadingMode(ThreadingMode.DEDICATED).dirsDeleteOnStart(deleteDirectoryOnStart).termBufferSparseFile(false).ipcTermBufferLength(ipcLength).publicationTermBufferLength(ipcLength).maxTermBufferLength(ipcLength).conductorIdleStrategy(new BusySpinIdleStrategy()).receiverIdleStrategy(new BusySpinIdleStrategy()).senderIdleStrategy(new BusySpinIdleStrategy());
        mediaDriver = MediaDriver.launchEmbedded(mediaDriverCtx);
        // set the variable since we are using a media driver directly
        mediaDriverDirectoryName = mediaDriver.aeronDirectoryName();
        log.info("Using media driver directory " + mediaDriver.aeronDirectoryName());
    }
    if (aeron == null)
        this.aeron = Aeron.connect(getContext());
    if (master) {
        if (this.callback == null) {
            ParameterServerUpdater updater = null;
            // instantiate with shape instead of just length
            switch(updateType) {
                case HOGWILD:
                    break;
                case SYNC:
                    updater = new SynchronousParameterUpdater(new InMemoryUpdateStorage(), new InMemoryNDArrayHolder(Ints.toArray(shape)), updatesPerEpoch);
                    break;
                case SOFTSYNC:
                    updater = new SoftSyncParameterUpdater();
                    break;
                case TIME_DELAYED:
                    break;
                case CUSTOM:
                    try {
                        updater = (ParameterServerUpdater) Class.forName(System.getProperty(CUSTOM_UPDATE_TYPE)).newInstance();
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                    break;
                default:
                    throw new IllegalStateException("Illegal opType of updater");
            }
            callback = new ParameterServerListener(Ints.toArray(shape), updater);
            parameterServerListener = (ParameterServerListener) callback;
        }
        // start an extra daemon for responding to get queries
        ParameterServerListener cast = (ParameterServerListener) callback;
        responder = AeronNDArrayResponder.startSubscriber(aeron, host, port + 1, cast.getUpdater().ndArrayHolder(), streamId + 1);
        log.info("Started responder on master node " + responder.connectionUrl());
    } else {
        String[] publishMasterUrlArr = publishMasterUrl.split(":");
        if (publishMasterUrlArr == null || publishMasterUrlArr.length < 2)
            throw new IllegalStateException("Please specify publish master url as host:port");
        callback = new PublishingListener(String.format("aeron:udp?endpoint=%s:%s", publishMasterUrlArr[0], publishMasterUrlArr[1]), Integer.parseInt(publishMasterUrlArr[2]), getContext());
    }
    log.info("Starting subscriber on " + host + ":" + port + " and stream " + streamId);
    AtomicBoolean running = new AtomicBoolean(true);
    // start a node
    subscriber = AeronNDArraySubscriber.startSubscriber(aeron, host, port, callback, streamId, running);
    while (!subscriber.launched()) {
        LockSupport.parkNanos(100000);
    }
    // Only schedule this if a remote server is available.
    if (CheckSocket.remotePortTaken(statusServerHost, statusServerPort, 10000)) {
        scheduledExecutorService = Executors.newScheduledThreadPool(1);
        final AtomicInteger failCount = new AtomicInteger(0);
        scheduledExecutorService.scheduleAtFixedRate(() -> {
            try {
                // 
                if (failCount.get() >= 3)
                    return;
                SubscriberState subscriberState = asState();
                JSONObject jsonObject = new JSONObject(objectMapper.writeValueAsString(subscriberState));
                String url = String.format("http://%s:%d/updatestatus/%d", statusServerHost, statusServerPort, streamId);
                HttpResponse<String> entity = Unirest.post(url).header("Content-Type", "application/json").body(jsonObject).asString();
            } catch (Exception e) {
                failCount.incrementAndGet();
                if (failCount.get() >= 3) {
                    log.warn("Failed to send update, shutting down likely?", e);
                }
            }
        }, 0, heartbeatMs, TimeUnit.MILLISECONDS);
    } else {
        log.info("No status server found. Will not send heartbeats. Specified host was " + statusServerHost + " and port was " + statusServerPort);
    }
    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
        close();
    }));
// set the server for the status of the master and slave nodes
}

Example 32 with MediaDriver

use of io.aeron.driver.MediaDriver in project nd4j by deeplearning4j.

the class RemoteParameterServerClientTests method before.

@Before
public void before() throws Exception {
    final MediaDriver.Context ctx = new MediaDriver.Context().threadingMode(ThreadingMode.DEDICATED).dirsDeleteOnStart(true).termBufferSparseFile(false).conductorIdleStrategy(new BusySpinIdleStrategy()).receiverIdleStrategy(new BusySpinIdleStrategy()).senderIdleStrategy(new BusySpinIdleStrategy());
    mediaDriver = MediaDriver.launchEmbedded(ctx);
    aeron = Aeron.connect(getContext());
    Thread t = new Thread(() -> {
        try {
            masterStatus.set(BackgroundDaemonStarter.startMaster(parameterLength, mediaDriver.aeronDirectoryName()));
        } catch (Exception e) {
            e.printStackTrace();
        }
    });
    t.start();
    log.info("Started master");
    Thread t2 = new Thread(() -> {
        try {
            slaveStatus.set(BackgroundDaemonStarter.startSlave(parameterLength, mediaDriver.aeronDirectoryName()));
        } catch (Exception e) {
            e.printStackTrace();
        }
    });
    t2.start();
    log.info("Started slave");
    // wait on the http servers
    Thread.sleep(30000);
}

Example 33 with MediaDriver

use of io.aeron.driver.MediaDriver in project aeron by real-logic.

the class ArchivingMediaDriver method launch.

/**
 * Launch a new {@link ArchivingMediaDriver} with provided contexts.
 *
 * @param driverCtx  for configuring the {@link MediaDriver}.
 * @param archiveCtx for configuring the {@link Archive}.
 * @return a new {@link ArchivingMediaDriver} with the provided contexts.
 */
public static ArchivingMediaDriver launch(final MediaDriver.Context driverCtx, final Archive.Context archiveCtx) {
    final MediaDriver driver = MediaDriver.launch(driverCtx);
    final Archive archive = Archive.launch(archiveCtx.mediaDriverAgentInvoker(driver.sharedAgentInvoker()).errorHandler(driverCtx.errorHandler()).errorCounter(driverCtx.systemCounters().get(SystemCounterDescriptor.ERRORS)));
    return new ArchivingMediaDriver(driver, archive);
}

Example 34 with MediaDriver

use of io.aeron.driver.MediaDriver in project aeron by real-logic.

the class BasicPublisher method main.

public static void main(final String[] args) throws Exception {
    System.out.println("Publishing to " + CHANNEL + " on stream Id " + STREAM_ID);
    // If configured to do so, create an embedded media driver within this application rather
    // than relying on an external one.
    final MediaDriver driver = EMBEDDED_MEDIA_DRIVER ? MediaDriver.launchEmbedded() : null;
    final Aeron.Context ctx = new Aeron.Context();
    if (EMBEDDED_MEDIA_DRIVER) {
        ctx.aeronDirectoryName(driver.aeronDirectoryName());
    }
    // clean up resources when this try block is finished
    try (Aeron aeron = Aeron.connect(ctx);
        Publication publication = aeron.addPublication(CHANNEL, STREAM_ID)) {
        for (long i = 0; i < NUMBER_OF_MESSAGES; i++) {
            final String message = "Hello World! " + i;
            final byte[] messageBytes = message.getBytes();
            BUFFER.putBytes(0, messageBytes);
            System.out.print("Offering " + i + "/" + NUMBER_OF_MESSAGES + " - ");
            final long result = publication.offer(BUFFER, 0, messageBytes.length);
            if (result < 0L) {
                if (result == Publication.BACK_PRESSURED) {
                    System.out.println("Offer failed due to back pressure");
                } else if (result == Publication.NOT_CONNECTED) {
                    System.out.println("Offer failed because publisher is not connected to subscriber");
                } else if (result == Publication.ADMIN_ACTION) {
                    System.out.println("Offer failed because of an administration action in the system");
                } else if (result == Publication.CLOSED) {
                    System.out.println("Offer failed publication is closed");
                    break;
                } else if (result == Publication.MAX_POSITION_EXCEEDED) {
                    System.out.println("Offer failed due to publication reaching max position");
                    break;
                } else {
                    System.out.println("Offer failed due to unknown reason");
                }
            } else {
                System.out.println("yay!");
            }
            if (!publication.isConnected()) {
                System.out.println("No active subscribers detected");
            }
            Thread.sleep(TimeUnit.SECONDS.toMillis(1));
        }
        System.out.println("Done sending.");
        if (LINGER_TIMEOUT_MS > 0) {
            System.out.println("Lingering for " + LINGER_TIMEOUT_MS + " milliseconds...");
            Thread.sleep(LINGER_TIMEOUT_MS);
        }
    }
    CloseHelper.quietClose(driver);
}

Example 35 with MediaDriver

use of io.aeron.driver.MediaDriver in project aeron by real-logic.

the class BasicSubscriber method main.

public static void main(final String[] args) {
    System.out.println("Subscribing to " + CHANNEL + " on stream Id " + STREAM_ID);
    final MediaDriver driver = EMBEDDED_MEDIA_DRIVER ? MediaDriver.launchEmbedded() : null;
    final Aeron.Context ctx = new Aeron.Context().availableImageHandler(SamplesUtil::printAvailableImage).unavailableImageHandler(SamplesUtil::printUnavailableImage);
    if (EMBEDDED_MEDIA_DRIVER) {
        ctx.aeronDirectoryName(driver.aeronDirectoryName());
    }
    final FragmentHandler fragmentHandler = SamplesUtil.printStringMessage(STREAM_ID);
    final AtomicBoolean running = new AtomicBoolean(true);
    // Register a SIGINT handler for graceful shutdown.
    SigInt.register(() -> running.set(false));
    // clean up resources when this try block is finished
    try (Aeron aeron = Aeron.connect(ctx);
        Subscription subscription = aeron.addSubscription(CHANNEL, STREAM_ID)) {
        SamplesUtil.subscriberLoop(fragmentHandler, FRAGMENT_COUNT_LIMIT, running).accept(subscription);
        System.out.println("Shutting down...");
    }
    CloseHelper.quietClose(driver);
}