Search in sources :

Example 26 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class LargeEventWriterTest method testUnexpectedErrors.

@Test(timeout = 5000)
public void testUnexpectedErrors() throws ConnectionFailedException, NoSuchSegmentException, AuthenticationException, SegmentSealedException {
    Segment segment = Segment.fromScopedName("foo/bar/1");
    MockConnectionFactoryImpl connectionFactory = new MockConnectionFactoryImpl();
    MockController controller = new MockController("localhost", 0, connectionFactory, false);
    ClientConnection connection = Mockito.mock(ClientConnection.class);
    PravegaNodeUri location = new PravegaNodeUri("localhost", 0);
    connectionFactory.provideConnection(location, connection);
    EmptyTokenProviderImpl tokenProvider = new EmptyTokenProviderImpl();
    EventWriterConfig config = EventWriterConfig.builder().initialBackoffMillis(0).build();
    ArrayList<ByteBuffer> events = new ArrayList<>();
    events.add(ByteBuffer.allocate(1));
    AtomicBoolean failed = new AtomicBoolean(false);
    AtomicBoolean succeeded = new AtomicBoolean(false);
    answerRequest(connectionFactory, connection, location, CreateTransientSegment.class, r -> new SegmentCreated(r.getRequestId(), "transient-segment"));
    answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).process(new InvalidEventNumber(argument.getWriterId(), argument.getEventNumber(), "stacktrace"));
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
            ByteBuf data = argument.getData();
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new DataAppended(argument.getRequestId(), argument.getWriterId(), argument.getEventNumber(), argument.getEventNumber() - 1, argument.getExpectedOffset() + data.readableBytes()));
            return null;
        }
    }).when(connection).send(any(ConditionalBlockEnd.class));
    answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
        return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
    });
    LargeEventWriter writer = new LargeEventWriter(writerId, controller, connectionFactory);
    writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
    assertTrue(failed.getAndSet(false));
    assertTrue(succeeded.getAndSet(false));
    answerRequest(connectionFactory, connection, location, CreateTransientSegment.class, r -> new SegmentCreated(r.getRequestId(), "transient-segment"));
    answerRequest(connectionFactory, connection, location, SetupAppend.class, r -> new AppendSetup(r.getRequestId(), segment.getScopedName(), r.getWriterId(), WireCommands.NULL_ATTRIBUTE_VALUE));
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
            failed.set(true);
            connectionFactory.getProcessor(location).process(new ConditionalCheckFailed(argument.getWriterId(), argument.getEventNumber(), argument.getRequestId()));
            return null;
        }
    }).doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            ConditionalBlockEnd argument = (ConditionalBlockEnd) invocation.getArgument(0);
            ByteBuf data = argument.getData();
            succeeded.set(true);
            connectionFactory.getProcessor(location).process(new DataAppended(argument.getRequestId(), argument.getWriterId(), argument.getEventNumber(), argument.getEventNumber() - 1, argument.getExpectedOffset() + data.readableBytes()));
            return null;
        }
    }).when(connection).send(any(ConditionalBlockEnd.class));
    answerRequest(connectionFactory, connection, location, MergeSegments.class, r -> {
        return new SegmentsMerged(r.getRequestId(), r.getSource(), r.getTarget(), -1);
    });
    writer = new LargeEventWriter(writerId, controller, connectionFactory);
    writer.writeLargeEvent(segment, events, tokenProvider, EventWriterConfig.builder().build());
    assertTrue(failed.getAndSet(false));
    assertTrue(succeeded.getAndSet(false));
}
Also used : ArrayList(java.util.ArrayList) ByteBuf(io.netty.buffer.ByteBuf) Segment(io.pravega.client.segment.impl.Segment) NoSuchSegment(io.pravega.shared.protocol.netty.WireCommands.NoSuchSegment) CreateTransientSegment(io.pravega.shared.protocol.netty.WireCommands.CreateTransientSegment) ConditionalCheckFailed(io.pravega.shared.protocol.netty.WireCommands.ConditionalCheckFailed) ConditionalBlockEnd(io.pravega.shared.protocol.netty.WireCommands.ConditionalBlockEnd) PravegaNodeUri(io.pravega.shared.protocol.netty.PravegaNodeUri) InvalidEventNumber(io.pravega.shared.protocol.netty.WireCommands.InvalidEventNumber) MockConnectionFactoryImpl(io.pravega.client.stream.mock.MockConnectionFactoryImpl) ClientConnection(io.pravega.client.connection.impl.ClientConnection) ByteBuffer(java.nio.ByteBuffer) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) EmptyTokenProviderImpl(io.pravega.client.security.auth.EmptyTokenProviderImpl) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SegmentCreated(io.pravega.shared.protocol.netty.WireCommands.SegmentCreated) Answer(org.mockito.stubbing.Answer) EventWriterConfig(io.pravega.client.stream.EventWriterConfig) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DataAppended(io.pravega.shared.protocol.netty.WireCommands.DataAppended) MockController(io.pravega.client.stream.mock.MockController) SegmentsMerged(io.pravega.shared.protocol.netty.WireCommands.SegmentsMerged) Test(org.junit.Test)

Example 27 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testAppendPipeliningWithSeal.

@Test(timeout = 15 * 1000)
public void testAppendPipeliningWithSeal() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data1 = new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] data2 = new byte[] { 1, 2, 3, 4, 5 };
    byte[] data3 = new byte[] { 1, 2, 3 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    InOrder connectionVerifier = Mockito.inOrder(connection);
    setupGetAttributes(streamSegmentName, clientId, store);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    connectionVerifier.verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    // Initiate one append.
    val store1 = new CompletableFuture<Long>();
    val ac1 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 1, 0, 1), store1);
    processor.append(new Append(streamSegmentName, clientId, 1, 1, Unpooled.wrappedBuffer(data1), null, requestId));
    verifyStoreAppend(ac1, data1);
    verify(tracker).updateOutstandingBytes(connection, data1.length, data1.length);
    // Second append will fail with StreamSegmentSealedException (this simulates having one append, immediately followed
    // by a Seal, then another append).
    val ac2 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 2, 1, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 2, 1, Unpooled.wrappedBuffer(data2), null, requestId));
    verifyStoreAppend(ac2, data2);
    verify(tracker).updateOutstandingBytes(connection, data2.length, data1.length + data2.length);
    verify(tracker).updateOutstandingBytes(connection, -data2.length, data1.length);
    // Third append should fail just like the second one. However this will have a higher event number than that one
    // and we use it to verify it won't prevent sending the SegmentIsSealed message or send it multiple times.
    val ac3 = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 3, 2, 1), Futures.failedFuture(new StreamSegmentSealedException(streamSegmentName)));
    processor.append(new Append(streamSegmentName, clientId, 3, 1, Unpooled.wrappedBuffer(data3), null, requestId));
    verifyStoreAppend(ac3, data3);
    verify(tracker).updateOutstandingBytes(connection, data3.length, data1.length + data3.length);
    // Complete the first one and verify it is acked properly.
    store1.complete(100L);
    connectionVerifier.verify(connection).send(new DataAppended(requestId, clientId, 1, 0L, 100L));
    // Verify that a SegmentIsSealed message is sent AFTER the ack from the first one (for the second append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 2L));
    // Verify that a second SegmentIsSealed is sent (for the third append).
    connectionVerifier.verify(connection).send(new WireCommands.SegmentIsSealed(requestId, streamSegmentName, "", 3L));
    verify(tracker).updateOutstandingBytes(connection, -data3.length, data1.length);
    verify(tracker).updateOutstandingBytes(connection, -data1.length, 0);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, 4, 3, 1), Futures.failedFuture(new IntentionalException(streamSegmentName)));
    AssertExtensions.assertThrows("append() accepted a new request after sending a SegmentIsSealed message.", () -> processor.append(new Append(streamSegmentName, clientId, 4, 1, Unpooled.wrappedBuffer(data1), null, requestId)), ex -> ex instanceof IllegalStateException);
    // Verify no more messages are sent over the connection.
    connectionVerifier.verifyNoMoreInteractions();
    verifyNoMoreInteractions(store);
}
Also used : lombok.val(lombok.val) InOrder(org.mockito.InOrder) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) IntentionalException(io.pravega.test.common.IntentionalException) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) CompletableFuture(java.util.concurrent.CompletableFuture) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) StreamSegmentSealedException(io.pravega.segmentstore.contracts.StreamSegmentSealedException) DataAppended(io.pravega.shared.protocol.netty.WireCommands.DataAppended) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) UUID(java.util.UUID) WireCommands(io.pravega.shared.protocol.netty.WireCommands) Test(org.junit.Test)

Example 28 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testAppendFails.

@Test
public void testAppendFails() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    val mockedRecorder = mock(SegmentStatsRecorder.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).statsRecorder(mockedRecorder).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new IntentionalException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    try {
        processor.append(new Append(streamSegmentName, clientId, data.length * 2, 1, Unpooled.wrappedBuffer(data), null, requestId));
        fail();
    } catch (IllegalStateException e) {
    // Expected
    }
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).close();
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verify(store, atMost(1)).append(any(), any(), any(), any());
    verifyNoMoreInteractions(connection);
    verify(mockedRecorder, never()).recordAppend(eq(streamSegmentName), eq(8L), eq(1), any());
}
Also used : lombok.val(lombok.val) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) IntentionalException(io.pravega.test.common.IntentionalException) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) UUID(java.util.UUID) Test(org.junit.Test)

Example 29 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testInvalidOffset.

@Test
public void testInvalidOffset() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(streamSegmentName, clientId, 100, store);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    try {
        processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
        fail();
    } catch (IllegalStateException e) {
    // expected
    }
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 100));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}
Also used : ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) UUID(java.util.UUID) Test(org.junit.Test)

Example 30 with AppendSetup

use of io.pravega.shared.protocol.netty.WireCommands.AppendSetup in project pravega by pravega.

the class AppendProcessorTest method testCancellationException.

@Test
public void testCancellationException() {
    String streamSegmentName = "scope/stream/testAppendSegment";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    ConnectionTracker tracker = mock(ConnectionTracker.class);
    @Cleanup AppendProcessor processor = AppendProcessor.defaultBuilder().store(store).connection(new TrackedConnection(connection, tracker)).build();
    setupGetAttributes(streamSegmentName, clientId, store);
    val ac = interceptAppend(store, streamSegmentName, updateEventNumber(clientId, data.length), Futures.failedFuture(new CancellationException()));
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null, requestId));
    verify(store).getAttributes(anyString(), eq(Collections.singleton(AttributeId.fromUUID(clientId))), eq(true), eq(AppendProcessor.TIMEOUT));
    verifyStoreAppend(ac, data);
    verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 0));
    verify(tracker).updateOutstandingBytes(connection, data.length, data.length);
    verify(connection).close();
    verify(tracker).updateOutstandingBytes(connection, -data.length, 0);
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}
Also used : lombok.val(lombok.val) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Cleanup(lombok.Cleanup) AppendSetup(io.pravega.shared.protocol.netty.WireCommands.AppendSetup) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) Append(io.pravega.shared.protocol.netty.Append) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) CancellationException(java.util.concurrent.CancellationException) SetupAppend(io.pravega.shared.protocol.netty.WireCommands.SetupAppend) UUID(java.util.UUID) Test(org.junit.Test)

Aggregations

AppendSetup (io.pravega.shared.protocol.netty.WireCommands.AppendSetup)54 Test (org.junit.Test)51 SetupAppend (io.pravega.shared.protocol.netty.WireCommands.SetupAppend)49 UUID (java.util.UUID)46 Append (io.pravega.shared.protocol.netty.Append)40 MockConnectionFactoryImpl (io.pravega.client.stream.mock.MockConnectionFactoryImpl)33 MockController (io.pravega.client.stream.mock.MockController)33 PravegaNodeUri (io.pravega.shared.protocol.netty.PravegaNodeUri)33 ClientConnection (io.pravega.client.connection.impl.ClientConnection)32 CompletableFuture (java.util.concurrent.CompletableFuture)26 Cleanup (lombok.Cleanup)26 ByteBuffer (java.nio.ByteBuffer)24 InOrder (org.mockito.InOrder)22 WireCommands (io.pravega.shared.protocol.netty.WireCommands)20 DataAppended (io.pravega.shared.protocol.netty.WireCommands.DataAppended)20 StreamSegmentStore (io.pravega.segmentstore.contracts.StreamSegmentStore)19 lombok.val (lombok.val)18 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)15 InvocationOnMock (org.mockito.invocation.InvocationOnMock)13 SegmentCreated (io.pravega.shared.protocol.netty.WireCommands.SegmentCreated)12