Examples with Header io.aeron.logbuffer.Header used on opensource projects

Example 31 with Header

use of io.aeron.logbuffer.Header in project aeron by real-logic.

the class MultiModuleSharedDriverTest method shouldSupportTwoSingleNodeClusters.

@Test
@InterruptAfter(20)
@SuppressWarnings({ "try", "methodlength" })
public void shouldSupportTwoSingleNodeClusters() {
    final MediaDriver.Context driverCtx = new MediaDriver.Context().threadingMode(ThreadingMode.SHARED).nameResolver(new RedirectingNameResolver(TestCluster.DEFAULT_NODE_MAPPINGS)).dirDeleteOnShutdown(false).dirDeleteOnStart(true);
    final Archive.Context archiveCtx = new Archive.Context().threadingMode(ArchiveThreadingMode.SHARED).archiveDir(new File(SystemUtil.tmpDirName(), "archive")).recordingEventsEnabled(false).deleteArchiveOnStart(true);
    try (ArchivingMediaDriver ignore = ArchivingMediaDriver.launch(driverCtx, archiveCtx)) {
        final ConsensusModule.Context moduleCtx0 = new ConsensusModule.Context().clusterId(0).deleteDirOnStart(true).clusterDir(new File(SystemUtil.tmpDirName(), "cluster-0-0")).logChannel("aeron:ipc?term-length=64k").logStreamId(100).serviceStreamId(104).consensusModuleStreamId(105).ingressChannel("aeron:udp?endpoint=localhost:9020").replicationChannel("aeron:udp?endpoint=localhost:0");
        final ClusteredServiceContainer.Context containerCtx0 = new ClusteredServiceContainer.Context().clusterId(moduleCtx0.clusterId()).clusteredService(new EchoService()).clusterDir(moduleCtx0.clusterDir()).serviceStreamId(moduleCtx0.serviceStreamId()).consensusModuleStreamId(moduleCtx0.consensusModuleStreamId());
        final ConsensusModule.Context moduleCtx1 = new ConsensusModule.Context().clusterId(1).deleteDirOnStart(true).clusterDir(new File(SystemUtil.tmpDirName(), "cluster-0-1")).logChannel("aeron:ipc?term-length=64k").logStreamId(200).serviceStreamId(204).consensusModuleStreamId(205).ingressChannel("aeron:udp?endpoint=localhost:9021").replicationChannel("aeron:udp?endpoint=localhost:0");
        final ClusteredServiceContainer.Context containerCtx1 = new ClusteredServiceContainer.Context().clusteredService(new EchoService()).clusterDir(moduleCtx1.clusterDir()).serviceStreamId(moduleCtx1.serviceStreamId()).consensusModuleStreamId(moduleCtx1.consensusModuleStreamId()).clusterId(moduleCtx1.clusterId());
        ConsensusModule consensusModule0 = null;
        ClusteredServiceContainer container0 = null;
        ConsensusModule consensusModule1 = null;
        ClusteredServiceContainer container1 = null;
        AeronCluster client0 = null;
        AeronCluster client1 = null;
        try {
            consensusModule0 = ConsensusModule.launch(moduleCtx0);
            consensusModule1 = ConsensusModule.launch(moduleCtx1);
            container0 = ClusteredServiceContainer.launch(containerCtx0);
            container1 = ClusteredServiceContainer.launch(containerCtx1);
            final MutableReference<String> egress = new MutableReference<>();
            final EgressListener egressListener = (clusterSessionId, timestamp, buffer, offset, length, header) -> egress.set(buffer.getStringWithoutLengthAscii(offset, length));
            client0 = AeronCluster.connect(new AeronCluster.Context().egressListener(egressListener).ingressChannel(moduleCtx0.ingressChannel()).egressChannel("aeron:udp?endpoint=localhost:0"));
            client1 = AeronCluster.connect(new AeronCluster.Context().egressListener(egressListener).ingressChannel(moduleCtx1.ingressChannel()).egressChannel("aeron:udp?endpoint=localhost:0"));
            echoMessage(client0, "Message 0", egress);
            echoMessage(client1, "Message 1", egress);
        } finally {
            systemTestWatcher.dataCollector().add(moduleCtx0.clusterDir());
            systemTestWatcher.dataCollector().add(moduleCtx1.clusterDir());
            CloseHelper.closeAll(client0, client1, consensusModule0, consensusModule1, container0, container1);
        }
    } finally {
        systemTestWatcher.dataCollector().add(driverCtx.aeronDirectory());
        systemTestWatcher.dataCollector().add(archiveCtx.archiveDir());
    }
}

Example 32 with Header

use of io.aeron.logbuffer.Header in project Aeron by real-logic.

the class FragmentedMessageTest method shouldReceivePublishedMessage.

@Theory
@Test(timeout = 10000)
public void shouldReceivePublishedMessage(final String channel, final ThreadingMode threadingMode) throws Exception {
    final MediaDriver.Context ctx = new MediaDriver.Context();
    ctx.threadingMode(threadingMode);
    final FragmentAssembler adapter = new FragmentAssembler(mockFragmentHandler);
    try (MediaDriver ignore = MediaDriver.launch(ctx);
        Aeron aeron = Aeron.connect();
        Publication publication = aeron.addPublication(channel, STREAM_ID);
        Subscription subscription = aeron.addSubscription(channel, STREAM_ID)) {
        final UnsafeBuffer srcBuffer = new UnsafeBuffer(new byte[ctx.mtuLength() * 4]);
        final int offset = 0;
        final int length = srcBuffer.capacity() / 4;
        for (int i = 0; i < 4; i++) {
            srcBuffer.setMemory(i * length, length, (byte) (65 + i));
        }
        while (publication.offer(srcBuffer, offset, srcBuffer.capacity()) < 0L) {
            Thread.yield();
        }
        final int expectedFragmentsBecauseOfHeader = 5;
        int numFragments = 0;
        do {
            numFragments += subscription.poll(adapter, FRAGMENT_COUNT_LIMIT);
        } while (numFragments < expectedFragmentsBecauseOfHeader);
        final ArgumentCaptor<DirectBuffer> bufferArg = ArgumentCaptor.forClass(DirectBuffer.class);
        final ArgumentCaptor<Header> headerArg = ArgumentCaptor.forClass(Header.class);
        verify(mockFragmentHandler, times(1)).onFragment(bufferArg.capture(), eq(offset), eq(srcBuffer.capacity()), headerArg.capture());
        final DirectBuffer capturedBuffer = bufferArg.getValue();
        for (int i = 0; i < srcBuffer.capacity(); i++) {
            assertThat("same at i=" + i, capturedBuffer.getByte(i), is(srcBuffer.getByte(i)));
        }
        assertThat(headerArg.getValue().flags(), is(END_FRAG_FLAG));
    } finally {
        ctx.deleteAeronDirectory();
    }
}

Example 33 with Header

use of io.aeron.logbuffer.Header in project nd4j by deeplearning4j.

the class RoutedTransport method init.

@Override
public void init(@NonNull VoidConfiguration voidConfiguration, @NonNull Clipboard clipboard, @NonNull NodeRole role, @NonNull String localIp, int localPort, short shardIndex) {
    this.nodeRole = role;
    this.clipboard = clipboard;
    this.voidConfiguration = voidConfiguration;
    this.shardIndex = shardIndex;
    this.messages = new LinkedBlockingQueue<>();
    // shutdown hook
    super.init(voidConfiguration, clipboard, role, localIp, localPort, shardIndex);
    setProperty("aeron.client.liveness.timeout", "30000000000");
    context = new Aeron.Context().publicationConnectionTimeout(30000000000L).driverTimeoutMs(30000).keepAliveInterval(100000000);
    driver = MediaDriver.launchEmbedded();
    context.aeronDirectoryName(driver.aeronDirectoryName());
    aeron = Aeron.connect(context);
    if (router == null)
        router = new InterleavedRouter();
    // we skip IPs assign process if they were defined externally
    if (port == 0) {
        ip = localIp;
        port = localPort;
    }
    unicastChannelUri = "aeron:udp?endpoint=" + ip + ":" + port;
    subscriptionForClients = aeron.addSubscription(unicastChannelUri, voidConfiguration.getStreamId());
    // clean shut down
    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
        CloseHelper.quietClose(aeron);
        CloseHelper.quietClose(driver);
        CloseHelper.quietClose(context);
        CloseHelper.quietClose(subscriptionForClients);
    }));
    messageHandlerForClients = new FragmentAssembler((buffer, offset, length, header) -> jointMessageHandler(buffer, offset, length, header));
    /*
            Now, regardless of current role,
             we set up publication channel to each shard
         */
    String shardChannelUri = null;
    String remoteIp = null;
    int remotePort = 0;
    for (String ip : voidConfiguration.getShardAddresses()) {
        if (ip.contains(":")) {
            shardChannelUri = "aeron:udp?endpoint=" + ip;
            String[] split = ip.split(":");
            remoteIp = split[0];
            remotePort = Integer.valueOf(split[1]);
        } else {
            shardChannelUri = "aeron:udp?endpoint=" + ip + ":" + voidConfiguration.getUnicastPort();
            remoteIp = ip;
            remotePort = voidConfiguration.getUnicastPort();
        }
        Publication publication = aeron.addPublication(shardChannelUri, voidConfiguration.getStreamId());
        RemoteConnection connection = RemoteConnection.builder().ip(remoteIp).port(remotePort).publication(publication).locker(new Object()).build();
        shards.add(connection);
    }
    if (nodeRole == NodeRole.SHARD)
        log.info("Initialized as [{}]; ShardIndex: [{}]; Own endpoint: [{}]", nodeRole, shardIndex, unicastChannelUri);
    else
        log.info("Initialized as [{}]; Own endpoint: [{}]", nodeRole, unicastChannelUri);
    switch(nodeRole) {
        case MASTER:
        case BACKUP:
            {
            }
        case SHARD:
            {
                /*
                    For unicast transport we want to have interconnects between all shards first of all, because we know their IPs in advance.
                    But due to design requirements, clients have the same first step, so it's kinda shared for all states :)
                 */
                /*
                    Next step is connections setup for backup nodes.
                    TODO: to be implemented
                 */
                addClient(ip, port);
            }
            break;
        case CLIENT:
            {
            /*
                    For Clients on unicast transport, we either set up connection to single Shard, or to multiple shards
                    But since this code is shared - we don't do anything here
                 */
            }
            break;
        default:
            throw new ND4JIllegalStateException("Unknown NodeRole being passed: " + nodeRole);
    }
    router.init(voidConfiguration, this);
    this.originatorId = HashUtil.getLongHash(this.getIp() + ":" + this.getPort());
}

Example 34 with Header

use of io.aeron.logbuffer.Header in project aeron by real-logic.

the class ClusterNodeRestartTest method launchService.

private void launchService(final boolean initialLaunch, final AtomicLong msgCounter) {
    final ClusteredService service = new StubClusteredService() {

        private int counterValue = 0;

        public void onSessionMessage(final long clusterSessionId, final long correlationId, final long timestampMs, final DirectBuffer buffer, final int offset, final int length, final Header header) {
            final int sentValue = buffer.getInt(offset);
            assertThat(sentValue, is(counterValue));
            counterValue++;
            serviceState.set(Integer.toString(counterValue));
            msgCounter.getAndIncrement();
        }

        public void onTakeSnapshot(final Publication snapshotPublication) {
            final ExpandableArrayBuffer buffer = new ExpandableArrayBuffer();
            int length = 0;
            buffer.putInt(length, counterValue);
            length += SIZE_OF_INT;
            length += buffer.putIntAscii(length, counterValue);
            snapshotPublication.offer(buffer, 0, length);
        }

        public void onLoadSnapshot(final Image snapshotImage) {
            while (true) {
                final int fragments = snapshotImage.poll((buffer, offset, length, header) -> {
                    counterValue = buffer.getInt(offset);
                    final String s = buffer.getStringWithoutLengthAscii(offset + SIZE_OF_INT, length - SIZE_OF_INT);
                    serviceState.set(s);
                }, 1);
                if (fragments == 1) {
                    break;
                }
                TestUtil.checkInterruptedStatus();
                Thread.yield();
            }
        }
    };
    container = null;
    container = ClusteredServiceContainer.launch(new ClusteredServiceContainer.Context().clusteredService(service).terminationHook(() -> {
    }).errorHandler(Throwable::printStackTrace).deleteDirOnStart(initialLaunch));
}

Example 35 with Header

use of io.aeron.logbuffer.Header in project Aeron by real-logic.

the class ReplaySessionTest method shouldCalculateBlockSizeBasedOnFullFragments.

@Test
public void shouldCalculateBlockSizeBasedOnFullFragments() throws IOException {
    context.recordChecksumBuffer(new UnsafeBuffer(ByteBuffer.allocateDirect(TERM_BUFFER_LENGTH)));
    final RecordingWriter writer = new RecordingWriter(RECORDING_ID, START_POSITION, SEGMENT_LENGTH, mockImage, context);
    writer.init();
    final UnsafeBuffer buffer = new UnsafeBuffer(allocateDirectAligned(TERM_BUFFER_LENGTH, 64));
    final DataHeaderFlyweight headerFwt = new DataHeaderFlyweight();
    final Header header = new Header(INITIAL_TERM_ID, Integer.numberOfLeadingZeros(TERM_BUFFER_LENGTH));
    header.buffer(buffer);
    recordFragment(writer, buffer, headerFwt, header, 0, 100, 11, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    recordFragment(writer, buffer, headerFwt, header, 128, 200, 22, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    recordFragment(writer, buffer, headerFwt, header, 352, 300, 33, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    recordFragment(writer, buffer, headerFwt, header, 672, 400, 44, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    recordFragment(writer, buffer, headerFwt, header, 1088, 500, 55, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    recordFragment(writer, buffer, headerFwt, header, 1600, 6000, 66, UNFRAGMENTED, HDR_TYPE_DATA, SESSION_ID);
    writer.close();
    final int sessionId = 555;
    final int streamId = 777;
    try (ReplaySession replaySession = replaySession(RECORDING_POSITION, 1_000_000, -1, mockReplayPub, mockControlSession, null, null)) {
        when(mockReplayPub.isClosed()).thenReturn(false);
        when(mockReplayPub.isConnected()).thenReturn(false);
        when(mockReplayPub.sessionId()).thenReturn(sessionId);
        when(mockReplayPub.streamId()).thenReturn(streamId);
        replaySession.doWork();
        assertEquals(replaySession.state(), ReplaySession.State.INIT);
        when(mockReplayPub.isConnected()).thenReturn(true);
        final UnsafeBuffer termBuffer = new UnsafeBuffer(allocateDirectAligned(4096, 64));
        mockPublication(mockReplayPub, termBuffer);
        assertEquals(2, replaySession.doWork());
        assertThat(messageCounter, is(1));
        validateFrame(termBuffer, 0, 100, 11, UNFRAGMENTED, sessionId, streamId);
        validateFrame(termBuffer, 128, 200, 22, UNFRAGMENTED, sessionId, streamId);
        validateFrame(termBuffer, 352, 300, 33, UNFRAGMENTED, sessionId, streamId);
        validateFrame(termBuffer, 672, 400, 44, UNFRAGMENTED, sessionId, streamId);
        validateFrame(termBuffer, 1088, 500, 55, UNFRAGMENTED, sessionId, streamId);
        verify(mockReplayPub, never()).appendPadding(anyInt());
    }
}