Search in sources :

Example 1 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testFailDurringFlush.

@Test(timeout = 10000)
public void testFailDurringFlush() throws ConnectionFailedException {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, "");
    output.reconnect();
    order.verify(connection).send(new SetupAppend(1, cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(1, SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Boolean> ack = new CompletableFuture<>();
    output.write(new PendingEvent(null, data, ack));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, Unpooled.wrappedBuffer(data), null));
    assertEquals(false, ack.isDone());
    Mockito.doThrow(new ConnectionFailedException()).when(connection).send(new WireCommands.KeepAlive());
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            cf.getProcessor(uri).appendSetup(new AppendSetup(3, SEGMENT, cid, 1));
            return null;
        }
    }).when(connection).send(new SetupAppend(3, cid, SEGMENT, ""));
    Async.testBlocking(() -> {
        output.flush();
    }, () -> {
        cf.getProcessor(uri).connectionDropped();
    });
    order.verify(connection).send(new SetupAppend(3, cid, SEGMENT, ""));
    assertEquals(true, ack.isDone());
}
Also used : ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InOrder(org.mockito.InOrder) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) PendingEvent(io.pravega.client.stream.impl.PendingEvent) InvocationOnMock(org.mockito.invocation.InvocationOnMock) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.netty.impl.ClientConnection) UUID(java.util.UUID) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) WireCommands(io.pravega.shared.protocol.netty.WireCommands) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) Test(org.junit.Test)

Example 2 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class ConditionalOutputStreamImpl method write.

@Override
public boolean write(ByteBuffer data, long expectedOffset) throws SegmentSealedException {
    Exceptions.checkNotClosed(closed.get(), this);
    synchronized (lock) {
        // Used to preserver order.
        long appendSequence = requestIdGenerator.get();
        return retrySchedule.retryWhen(e -> {
            Throwable cause = Exceptions.unwrap(e);
            boolean hasTokenExpired = cause instanceof TokenExpiredException;
            if (hasTokenExpired) {
                this.tokenProvider.signalTokenExpired();
            }
            return cause instanceof Exception && (hasTokenExpired || cause instanceof ConnectionFailedException);
        }).run(() -> {
            if (client == null || client.isClosed()) {
                client = new RawClient(controller, connectionPool, segmentId);
                long requestId = client.getFlow().getNextSequenceNumber();
                log.debug("Setting up appends on segment {} for ConditionalOutputStream with writer id {}", segmentId, writerId);
                CompletableFuture<Reply> reply = tokenProvider.retrieveToken().thenCompose(token -> {
                    SetupAppend setup = new SetupAppend(requestId, writerId, segmentId.getScopedName(), token);
                    return client.sendRequest(requestId, setup);
                });
                AppendSetup appendSetup = transformAppendSetup(reply.join());
                if (appendSetup.getLastEventNumber() >= appendSequence) {
                    return true;
                }
            }
            long requestId = client.getFlow().getNextSequenceNumber();
            val request = new ConditionalAppend(writerId, appendSequence, expectedOffset, new Event(Unpooled.wrappedBuffer(data)), requestId);
            val reply = client.sendRequest(requestId, request);
            return transformDataAppended(reply.join());
        });
    }
}
Also used : Event(io.pravega.shared.protocol.netty.WireCommands.Event) TokenExpiredException(io.pravega.auth.TokenExpiredException) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) Reply(io.pravega.shared.protocol.netty.Reply) Exceptions(io.pravega.common.Exceptions) ConditionalCheckFailed(io.pravega.shared.protocol.netty.WireCommands.ConditionalCheckFailed) RequiredArgsConstructor(lombok.RequiredArgsConstructor) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CompletableFuture(java.util.concurrent.CompletableFuture) Supplier(java.util.function.Supplier) ByteBuffer(java.nio.ByteBuffer) Unpooled(io.netty.buffer.Unpooled) RawClient(io.pravega.client.connection.impl.RawClient) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) AuthTokenCheckFailed(io.pravega.shared.protocol.netty.WireCommands.AuthTokenCheckFailed) ConditionalAppend(io.pravega.shared.protocol.netty.WireCommands.ConditionalAppend) RetryWithBackoff(io.pravega.common.util.Retry.RetryWithBackoff) ConnectionPool(io.pravega.client.connection.impl.ConnectionPool) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) lombok.val(lombok.val) AuthenticationException(io.pravega.auth.AuthenticationException) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) GuardedBy(javax.annotation.concurrent.GuardedBy) WrongHost(io.pravega.shared.protocol.netty.WireCommands.WrongHost) DelegationTokenProvider(io.pravega.client.security.auth.DelegationTokenProvider) AtomicLong(java.util.concurrent.atomic.AtomicLong) InvalidEventNumber(io.pravega.shared.protocol.netty.WireCommands.InvalidEventNumber) Slf4j(lombok.extern.slf4j.Slf4j) VisibleForTesting(com.google.common.annotations.VisibleForTesting) DataAppended(io.pravega.shared.protocol.netty.WireCommands.DataAppended) SegmentIsSealed(io.pravega.shared.protocol.netty.WireCommands.SegmentIsSealed) Controller(io.pravega.client.control.impl.Controller) lombok.val(lombok.val) ConditionalAppend(io.pravega.shared.protocol.netty.WireCommands.ConditionalAppend) RawClient(io.pravega.client.connection.impl.RawClient) TokenExpiredException(io.pravega.auth.TokenExpiredException) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) AuthenticationException(io.pravega.auth.AuthenticationException) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) TokenExpiredException(io.pravega.auth.TokenExpiredException) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) Reply(io.pravega.shared.protocol.netty.Reply) Event(io.pravega.shared.protocol.netty.WireCommands.Event) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException)

Example 3 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testSealedAfterFlush.

@Test(timeout = 10000)
public void testSealedAfterFlush() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @SuppressWarnings("resource") SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> ack = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, ack.isDone());
    AssertExtensions.assertBlocks(() -> {
        AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush());
    }, () -> {
        cf.getProcessor(uri).segmentIsSealed(new WireCommands.SegmentIsSealed(output.getRequestId(), SEGMENT, "SomeException", 1));
        output.getUnackedEventsOnSeal();
    });
    AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.flush());
}
Also used : InOrder(org.mockito.InOrder) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) Test(org.junit.Test)

Example 4 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testCloseDuringTransactionAbort.

@Test(timeout = 10000)
public void testCloseDuringTransactionAbort() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    cf.setExecutor(executorService());
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(TXN_SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, TXN_SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), TXN_SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    // Write an Event.
    CompletableFuture<Void> ack1 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack1));
    order.verify(connection).send(new Append(TXN_SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    // writer is not complete until a response from Segment Store is received.
    assertFalse(ack1.isDone());
    // Simulate a NoSuchSegment response from SegmentStore due to a Transaction abort.
    cf.getProcessor(uri).noSuchSegment(new WireCommands.NoSuchSegment(output.getRequestId(), TXN_SEGMENT, "SomeException", -1L));
    // Trigger a second write.
    CompletableFuture<Void> ack2 = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack2));
    // Closing the Segment writer should cause a SegmentSealedException.
    AssertExtensions.assertThrows(SegmentSealedException.class, () -> output.close());
}
Also used : InOrder(org.mockito.InOrder) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) Test(org.junit.Test)

Example 5 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class SegmentOutputStreamTest method testFailDuringFlush.

@Test(timeout = 10000)
public void testFailDuringFlush() throws Exception {
    UUID cid = UUID.randomUUID();
    PravegaNodeUri uri = new PravegaNodeUri("endpoint", SERVICE_PORT);
    MockConnectionFactoryImpl cf = new MockConnectionFactoryImpl();
    ScheduledExecutorService executor = mock(ScheduledExecutorService.class);
    // Ensure task submitted to executor is run inline.
    implementAsDirectExecutor(executor);
    cf.setExecutor(executor);
    MockController controller = new MockController(uri.getEndpoint(), uri.getPort(), cf, true);
    ClientConnection connection = mock(ClientConnection.class);
    cf.provideConnection(uri, connection);
    InOrder order = Mockito.inOrder(connection);
    @Cleanup SegmentOutputStreamImpl output = new SegmentOutputStreamImpl(SEGMENT, true, controller, cf, cid, segmentSealedCallback, RETRY_SCHEDULE, DelegationTokenProviderFactory.createWithEmptyToken());
    output.reconnect();
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 0));
    ByteBuffer data = getBuffer("test");
    CompletableFuture<Void> ack = new CompletableFuture<>();
    output.write(PendingEvent.withoutHeader(null, data, ack));
    order.verify(connection).send(new Append(SEGMENT, cid, 1, 1, Unpooled.wrappedBuffer(data), null, output.getRequestId()));
    assertEquals(false, ack.isDone());
    final CountDownLatch connectionDroppedLatch = new CountDownLatch(1);
    Mockito.doThrow(new ConnectionFailedException()).when(connection).send(new WireCommands.KeepAlive());
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // The segment writer will try to reconnect once the connection is failed post sending a KeepAlive.
            // enable a response for AppendSetup only after the connection dropped is dropped.
            connectionDroppedLatch.await();
            cf.getProcessor(uri).appendSetup(new AppendSetup(output.getRequestId(), SEGMENT, cid, 1));
            return null;
        }
    }).when(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    AssertExtensions.assertBlocks(() -> {
        output.flush();
    }, () -> {
        cf.getProcessor(uri).connectionDropped();
        connectionDroppedLatch.countDown();
    });
    order.verify(connection).send(new SetupAppend(output.getRequestId(), cid, SEGMENT, ""));
    assertEquals(true, ack.isDone());
}
Also used : ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) InOrder(org.mockito.InOrder) CountDownLatch(java.util.concurrent.CountDownLatch) Cleanup(lombok.Cleanup) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) InvocationOnMock(org.mockito.invocation.InvocationOnMock) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) MockController(io.pravega.client.stream.mock.MockController) ClientConnection(io.pravega.client.connection.impl.ClientConnection) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) ConnectionFailedException(io.pravega.shared.protocol.netty.ConnectionFailedException) Test(org.junit.Test)

Aggregations

AppendSetup (io.pravega.shared.protocol.netty.WireCommands.AppendSetup)54 Test (org.junit.Test)51 SetupAppend (io.pravega.shared.protocol.netty.WireCommands.SetupAppend)49 UUID (java.util.UUID)46 Append (io.pravega.shared.protocol.netty.Append)40 MockConnectionFactoryImpl (io.pravega.client.stream.mock.MockConnectionFactoryImpl)33 MockController (io.pravega.client.stream.mock.MockController)33 PravegaNodeUri (io.pravega.shared.protocol.netty.PravegaNodeUri)33 ClientConnection (io.pravega.client.connection.impl.ClientConnection)32 CompletableFuture (java.util.concurrent.CompletableFuture)26 Cleanup (lombok.Cleanup)26 ByteBuffer (java.nio.ByteBuffer)24 InOrder (org.mockito.InOrder)22 WireCommands (io.pravega.shared.protocol.netty.WireCommands)20 DataAppended (io.pravega.shared.protocol.netty.WireCommands.DataAppended)20 StreamSegmentStore (io.pravega.segmentstore.contracts.StreamSegmentStore)19 lombok.val (lombok.val)18 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)15 InvocationOnMock (org.mockito.invocation.InvocationOnMock)13 SegmentCreated (io.pravega.shared.protocol.netty.WireCommands.SegmentCreated)12