Examples with StreamSegmentStore io.pravega.segmentstore.contracts.StreamSegmentStore used on opensource projects

Example 46 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class AppendProcessorTest method testDelayedDataAppended.

/**
 * Test to ensure newer appends are processed only after successfully sending the DataAppended acknowledgement
 * back to client. This test tests the following:
 * - If sending first DataAppended is blocked, ensure future appends are not written to store.
 * - Once the first DataAppended is sent ensure the remaining appends are written to store and DataAppended ack'ed
 * back.
 */
@Test(timeout = 15 * 1000)
public void testDelayedDataAppended() throws Exception {
    ReusableLatch firstStoreAppendInvoked = new ReusableLatch();
    ReusableLatch completeFirstDataAppendedAck = new ReusableLatch();
    ReusableLatch secondStoreAppendInvoked = new ReusableLatch();
    @Cleanup("shutdownNow") ScheduledExecutorService nettyExecutor = ExecutorServiceHelpers.newScheduledThreadPool(1, "Netty-threadPool");
    String streamSegmentName = "testDelayedAppend";
    UUID clientId = UUID.randomUUID();
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    // Ensure the first DataAppended is hung/delayed.
    doAnswer(invocation -> {
        firstStoreAppendInvoked.release();
        // wait, simulating a hung/delayed dataAppended acknowledgement.
        completeFirstDataAppendedAck.await();
        return null;
    }).doAnswer(invocation -> {
        secondStoreAppendInvoked.release();
        return null;
    }).when(connection).send(any(DataAppended.class));
    AppendProcessor processor = new AppendProcessor(store, connection, new FailingRequestProcessor(), null);
    CompletableFuture<SegmentProperties> propsFuture = CompletableFuture.completedFuture(StreamSegmentInformation.builder().name(streamSegmentName).build());
    when(store.getStreamSegmentInfo(streamSegmentName, true, AppendProcessor.TIMEOUT)).thenReturn(propsFuture);
    processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, ""));
    verify(store).getStreamSegmentInfo(streamSegmentName, true, AppendProcessor.TIMEOUT);
    CompletableFuture<Void> result = CompletableFuture.completedFuture(null);
    int eventCount = 100;
    when(store.append(streamSegmentName, data, updateEventNumber(clientId, 100, SegmentMetadata.NULL_ATTRIBUTE_VALUE, eventCount), AppendProcessor.TIMEOUT)).thenReturn(result);
    // Trigger the first append, here the sending of DataAppended ack will be delayed/hung.
    nettyExecutor.submit(() -> processor.append(new Append(streamSegmentName, clientId, 100, eventCount, Unpooled.wrappedBuffer(data), null)));
    firstStoreAppendInvoked.await();
    verify(store).append(streamSegmentName, data, updateEventNumber(clientId, 100, SegmentMetadata.NULL_ATTRIBUTE_VALUE, eventCount), AppendProcessor.TIMEOUT);
    /* Trigger the next append. This should be completed immediately and should not cause a store.append to be
        invoked as the previous DataAppended ack is still not sent. */
    processor.append(new Append(streamSegmentName, clientId, 200, eventCount, Unpooled.wrappedBuffer(data), null));
    // Since the first Ack was never sent the next append should not be written to the store.
    verifyNoMoreInteractions(store);
    // Setup mock for check behaviour after the delayed/hung dataAppended completes.
    when(store.append(streamSegmentName, data, updateEventNumber(clientId, 200, 100, eventCount), AppendProcessor.TIMEOUT)).thenReturn(result);
    // Now ensure the dataAppended sent
    completeFirstDataAppendedAck.release();
    // wait until the next store append is invoked.
    secondStoreAppendInvoked.await();
    // Verify that the next store append invoked.
    verify(store).append(streamSegmentName, data, updateEventNumber(clientId, 200, 100, eventCount), AppendProcessor.TIMEOUT);
    // Verify two DataAppended acks are sent out.
    verify(connection, times(2)).send(any(DataAppended.class));
    verify(connection).send(new DataAppended(clientId, 100, Long.MIN_VALUE));
    verify(connection).send(new DataAppended(clientId, 200, 100));
}

Example 47 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegmentTruncated.

@Test(timeout = 20000)
public void testReadSegmentTruncated() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Truncated, 0, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    StreamSegmentInformation info = StreamSegmentInformation.builder().name(streamSegmentName).length(1234).startOffset(123).build();
    when(store.getStreamSegmentInfo(streamSegmentName, false, PravegaRequestProcessor.TIMEOUT)).thenReturn(CompletableFuture.completedFuture(info));
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(store).getStreamSegmentInfo(streamSegmentName, false, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentIsTruncated(0, streamSegmentName, info.getStartOffset()));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 48 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegment.

@Test(timeout = 20000)
public void testReadSegment() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 };
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.Cache, 0, readLength);
    entry1.complete(new ReadResultEntryContents(new ByteArrayInputStream(data), data.length));
    TestReadResultEntry entry2 = new TestReadResultEntry(ReadResultEntryType.Future, data.length, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    results.add(entry2);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, true, false, ByteBuffer.wrap(data)));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
    entry2.complete(new ReadResultEntryContents(new ByteArrayInputStream(data), data.length));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 49 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testReadSegmentEmptySealed.

@Test(timeout = 20000)
public void testReadSegmentEmptySealed() {
    // Set up PravegaRequestProcessor instance to execute read segment request against
    String streamSegmentName = "testReadSegment";
    int readLength = 1000;
    StreamSegmentStore store = mock(StreamSegmentStore.class);
    ServerConnection connection = mock(ServerConnection.class);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    TestReadResultEntry entry1 = new TestReadResultEntry(ReadResultEntryType.EndOfStreamSegment, 0, readLength);
    List<ReadResultEntry> results = new ArrayList<>();
    results.add(entry1);
    CompletableFuture<ReadResult> readResult = new CompletableFuture<>();
    readResult.complete(new TestReadResult(0, readLength, results));
    when(store.read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT)).thenReturn(readResult);
    // Execute and Verify readSegment calling stack in connection and store is executed as design.
    processor.readSegment(new WireCommands.ReadSegment(streamSegmentName, 0, readLength, ""));
    verify(store).read(streamSegmentName, 0, readLength, PravegaRequestProcessor.TIMEOUT);
    verify(connection).send(new WireCommands.SegmentRead(streamSegmentName, 0, false, true, ByteBuffer.wrap(new byte[0])));
    verifyNoMoreInteractions(connection);
    verifyNoMoreInteractions(store);
}

Example 50 with StreamSegmentStore

use of io.pravega.segmentstore.contracts.StreamSegmentStore in project pravega by pravega.

the class PravegaRequestProcessorTest method testTransaction.

@Test(timeout = 20000)
public void testTransaction() throws Exception {
    String streamSegmentName = "testTxn";
    UUID txnid = UUID.randomUUID();
    @Cleanup ServiceBuilder serviceBuilder = newInlineExecutionInMemoryBuilder(getBuilderConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    ServerConnection connection = mock(ServerConnection.class);
    InOrder order = inOrder(connection);
    PravegaRequestProcessor processor = new PravegaRequestProcessor(store, connection);
    processor.createSegment(new WireCommands.CreateSegment(0, streamSegmentName, WireCommands.CreateSegment.NO_SCALE, 0, ""));
    order.verify(connection).send(new WireCommands.SegmentCreated(0, streamSegmentName));
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 2, store));
    order.verify(connection).send(new WireCommands.TransactionCreated(1, streamSegmentName, txnid));
    order.verify(connection).send(Mockito.argThat(t -> {
        return t instanceof TransactionInfo && ((TransactionInfo) t).exists();
    }));
    processor.commitTransaction(new WireCommands.CommitTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionCommitted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    txnid = UUID.randomUUID();
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    order.verify(connection).send(new WireCommands.TransactionCreated(1, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    order.verify(connection).send(Mockito.argThat(t -> {
        return t instanceof TransactionInfo && ((TransactionInfo) t).exists();
    }));
    processor.abortTransaction(new WireCommands.AbortTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionAborted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    // Verify the case when the transaction segment is already sealed. This simulates the case when the process
    // crashed after sealing, but before issuing the merge.
    txnid = UUID.randomUUID();
    processor.createTransaction(new WireCommands.CreateTransaction(1, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 1, store));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(2, streamSegmentName, txnid, ""));
    assertTrue(append(StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid), 2, store));
    // Seal the transaction in the SegmentStore.
    String txnName = StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid);
    store.sealStreamSegment(txnName, Duration.ZERO).join();
    processor.commitTransaction(new WireCommands.CommitTransaction(3, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.TransactionCommitted(3, streamSegmentName, txnid));
    processor.getTransactionInfo(new WireCommands.GetTransactionInfo(4, streamSegmentName, txnid, ""));
    order.verify(connection).send(new WireCommands.NoSuchSegment(4, StreamSegmentNameUtils.getTransactionNameFromId(streamSegmentName, txnid)));
    order.verifyNoMoreInteractions();
}