Examples with ScalingPolicy io.pravega.client.stream.ScalingPolicy used on opensource projects

Example 31 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataTasksTest method retentionPolicyUpdateTest.

@Test(timeout = 30000)
public void retentionPolicyUpdateTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    String stream = "test";
    final StreamConfiguration noRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).build();
    // add stream without retention policy
    streamMetadataTasks.createStreamBody(SCOPE, stream, noRetentionConfig, System.currentTimeMillis()).join();
    String scopedStreamName = String.format("%s/%s", SCOPE, stream);
    // verify that stream is not added to bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    UpdateStreamTask task = new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    final RetentionPolicy retentionPolicy = RetentionPolicy.builder().retentionType(RetentionPolicy.RetentionType.TIME).retentionParam(Duration.ofMinutes(60).toMillis()).build();
    final StreamConfiguration withRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).retentionPolicy(retentionPolicy).build();
    // now update stream with a retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, withRetentionConfig, null, executor).join();
    UpdateStreamEvent update = new UpdateStreamEvent(SCOPE, stream);
    task.execute(update).join();
    // verify that bucket has the stream.
    assertTrue(streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    // update stream such that stream is updated with null retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, noRetentionConfig, null, executor).join();
    task.execute(update).join();
    // verify that the stream is no longer present in the bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
}

Example 32 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method commitAbortTests.

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void commitAbortTests() {
    // Create mock writer objects.
    final List<CompletableFuture<Void>> commitWriterResponses = getWriteResultSequence(5);
    final List<CompletableFuture<Void>> abortWriterResponses = getWriteResultSequence(5);
    EventStreamWriter<CommitEvent> commitWriter = Mockito.mock(EventStreamWriter.class);
    Mockito.when(commitWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(commitWriterResponses));
    EventStreamWriter<AbortEvent> abortWriter = Mockito.mock(EventStreamWriter.class);
    Mockito.when(abortWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(abortWriterResponses));
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Create ControllerService.
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Create stream and scope
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
    // Create 2 transactions
    final long lease = 5000;
    final long scaleGracePeriod = 10000;
    VersionedTransactionData txData1 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    VersionedTransactionData txData2 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    // Commit the first one
    TxnStatus status = txnTasks.commitTxn(SCOPE, STREAM, txData1.getId(), null).join();
    Assert.assertEquals(TxnStatus.COMMITTING, status);
    // Abort the second one
    status = txnTasks.abortTxn(SCOPE, STREAM, txData2.getId(), txData2.getVersion(), null).join();
    Assert.assertEquals(TxnStatus.ABORTING, status);
}

Example 33 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method partialTxnCreationTest.

@Test(timeout = 10000)
public void partialTxnCreationTest() {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, SegmentHelperMock.getFailingSegmentHelperMock(), executor, "host", connectionFactory, this.authEnabled, "secret");
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Create ControllerService.
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Create stream and scope
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
    // Create partial transaction
    final long lease = 10000;
    final long scaleGracePeriod = 10000;
    AssertExtensions.assertThrows("Transaction creation fails, although a new txn id gets added to the store", txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null), e -> e instanceof RuntimeException);
    // Ensure that exactly one transaction is active on the stream.
    Set<UUID> txns = streamStore.getActiveTxns(SCOPE, STREAM, null, executor).join().keySet();
    assertEquals(1, txns.size());
    // Ensure that transaction state is OPEN.
    UUID txn1 = txns.stream().findFirst().get();
    assertEquals(TxnStatus.OPEN, streamStore.transactionStatus(SCOPE, STREAM, txn1, null, executor).join());
    // Ensure that timeout service knows about the transaction.
    assertTrue(txnTasks.getTimeoutService().containsTxn(SCOPE, STREAM, txn1));
}

Example 34 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method idempotentOperationsTests.

@Test(timeout = 10000)
public void idempotentOperationsTests() throws CheckpointStoreException, InterruptedException {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    EventStreamReader<CommitEvent> commitReader = commitWriter.getReader();
    EventStreamReader<AbortEvent> abortReader = abortWriter.getReader();
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Create stream and scope
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis()).join());
    // Create 2 transactions
    final long lease = 5000;
    final long scaleGracePeriod = 10000;
    VersionedTransactionData txData1 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    VersionedTransactionData txData2 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    UUID tx1 = txData1.getId();
    UUID tx2 = txData2.getId();
    int tx2Version = txData2.getVersion();
    // Commit the first one
    Assert.assertEquals(TxnStatus.COMMITTING, txnTasks.commitTxn(SCOPE, STREAM, tx1, null).join());
    // Ensure that transaction state is COMMITTING.
    assertEquals(TxnStatus.COMMITTING, streamStore.transactionStatus(SCOPE, STREAM, tx1, null, executor).join());
    // Abort the second one
    Assert.assertEquals(TxnStatus.ABORTING, txnTasks.abortTxn(SCOPE, STREAM, tx2, tx2Version, null).join());
    // Ensure that transactions state is ABORTING.
    assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx2, null, executor).join());
    // Ensure that commit (resp. abort) transaction tasks are idempotent
    // when transaction is in COMMITTING state (resp. ABORTING state).
    assertEquals(TxnStatus.COMMITTING, txnTasks.commitTxn(SCOPE, STREAM, tx1, null).join());
    assertEquals(TxnStatus.ABORTING, txnTasks.abortTxn(SCOPE, STREAM, tx2, null, null).join());
    // Create commit and abort event processors.
    ConnectionFactory connectionFactory = Mockito.mock(ConnectionFactory.class);
    BlockingQueue<CommitEvent> processedCommitEvents = new LinkedBlockingQueue<>();
    BlockingQueue<AbortEvent> processedAbortEvents = new LinkedBlockingQueue<>();
    createEventProcessor("commitRG", "commitStream", commitReader, commitWriter, () -> new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedCommitEvents));
    createEventProcessor("abortRG", "abortStream", abortReader, abortWriter, () -> new ConcurrentEventProcessor<>(new AbortRequestHandler(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedAbortEvents), executor));
    // Wait until the commit event is processed and ensure that the txn state is COMMITTED.
    CommitEvent commitEvent = processedCommitEvents.take();
    assertEquals(tx1, commitEvent.getTxid());
    assertEquals(TxnStatus.COMMITTED, streamStore.transactionStatus(SCOPE, STREAM, tx1, null, executor).join());
    // Wait until the abort event is processed and ensure that the txn state is ABORTED.
    AbortEvent abortEvent = processedAbortEvents.take();
    assertEquals(tx2, abortEvent.getTxid());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx2, null, executor).join());
    // Ensure that commit (resp. abort) transaction tasks are idempotent
    // even after transaction is committed (resp. aborted)
    assertEquals(TxnStatus.COMMITTED, txnTasks.commitTxn(SCOPE, STREAM, tx1, null).join());
    assertEquals(TxnStatus.ABORTED, txnTasks.abortTxn(SCOPE, STREAM, tx2, null, null).join());
}

Example 35 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class TaskTest method testStreamTaskSweeping.

@Test(timeout = 10000)
public void testStreamTaskSweeping() {
    final String stream = "testPartialCreationStream";
    final String deadHost = "deadHost";
    final int initialSegments = 2;
    final ScalingPolicy policy1 = ScalingPolicy.fixed(initialSegments);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final ArrayList<Integer> sealSegments = new ArrayList<>();
    sealSegments.add(0);
    final ArrayList<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.25));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.25, 0.5));
    // Create objects.
    StreamMetadataTasks mockStreamTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelperMock, executor, deadHost, Mockito.mock(ConnectionFactory.class), false, "");
    mockStreamTasks.setCreateIndexOnlyMode();
    TaskSweeper sweeper = new TaskSweeper(taskMetadataStore, HOSTNAME, executor, streamMetadataTasks);
    // Create stream test.
    completePartialTask(mockStreamTasks.createStream(SCOPE, stream, configuration1, System.currentTimeMillis()), deadHost, sweeper);
    Assert.assertEquals(initialSegments, streamStore.getActiveSegments(SCOPE, stream, null, executor).join().size());
    List<StreamConfiguration> streams = streamStore.listStreamsInScope(SCOPE).join();
    Assert.assertTrue(streams.stream().allMatch(x -> !x.getStreamName().equals(stream)));
}