Examples with StreamConfiguration io.pravega.client.stream.StreamConfiguration used on opensource projects

Example 11 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method failOverTests.

@Test
public void failOverTests() throws CheckpointStoreException, InterruptedException {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    EventStreamReader<CommitEvent> commitReader = commitWriter.getReader();
    EventStreamReader<AbortEvent> abortReader = abortWriter.getReader();
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    // Create test scope and stream.
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis()).join());
    // Set up txn task for creating transactions from a failedHost.
    StreamTransactionMetadataTasks failedTxnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "failedHost", connectionFactory, false, "");
    failedTxnTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    // Create 3 transactions from failedHost.
    VersionedTransactionData tx1 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx2 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx3 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    // Ping another txn from failedHost.
    UUID txnId = UUID.randomUUID();
    streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 30000, 30000, null, executor).join();
    PingTxnStatus pingStatus = failedTxnTasks.pingTxn(SCOPE, STREAM, txnId, 10000, null).join();
    VersionedTransactionData tx4 = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    // Validate versions of all txn
    Assert.assertEquals(0, tx1.getVersion());
    Assert.assertEquals(0, tx2.getVersion());
    Assert.assertEquals(0, tx3.getVersion());
    Assert.assertEquals(1, tx4.getVersion());
    Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
    // Validate the txn index.
    Assert.assertEquals(1, streamStore.listHostsOwningTxn().join().size());
    // Change state of one txn to COMMITTING.
    TxnStatus txnStatus2 = streamStore.sealTransaction(SCOPE, STREAM, tx2.getId(), true, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.COMMITTING, txnStatus2);
    // Change state of another txn to ABORTING.
    TxnStatus txnStatus3 = streamStore.sealTransaction(SCOPE, STREAM, tx3.getId(), false, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.ABORTING, txnStatus3);
    // Create transaction tasks for sweeping txns from failedHost.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    TxnSweeper txnSweeper = new TxnSweeper(streamStore, txnTasks, 100, executor);
    // Before initializing, txnSweeper.sweepFailedHosts would throw an error
    AssertExtensions.assertThrows("IllegalStateException before initialization", txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")), ex -> ex instanceof IllegalStateException);
    // Initialize stream writers.
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Validate that txnTasks is ready.
    assertTrue(txnTasks.isReady());
    // Sweep txns that were being managed by failedHost.
    txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")).join();
    // Validate that sweeping completes correctly.
    Assert.assertEquals(0, streamStore.listHostsOwningTxn().join().size());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.COMMITTING, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
    // Create commit and abort event processors.
    ConnectionFactory connectionFactory = Mockito.mock(ConnectionFactory.class);
    BlockingQueue<CommitEvent> processedCommitEvents = new LinkedBlockingQueue<>();
    BlockingQueue<AbortEvent> processedAbortEvents = new LinkedBlockingQueue<>();
    createEventProcessor("commitRG", "commitStream", commitReader, commitWriter, () -> new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedCommitEvents));
    createEventProcessor("abortRG", "abortStream", abortReader, abortWriter, () -> new ConcurrentEventProcessor<>(new AbortRequestHandler(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedAbortEvents), executor));
    // Wait until the commit event is processed and ensure that the txn state is COMMITTED.
    CommitEvent commitEvent = processedCommitEvents.take();
    assertEquals(tx2.getId(), commitEvent.getTxid());
    assertEquals(TxnStatus.COMMITTED, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    // Wait until 3 abort events are processed and ensure that the txn state is ABORTED.
    Predicate<AbortEvent> predicate = event -> event.getTxid().equals(tx1.getId()) || event.getTxid().equals(tx3.getId()) || event.getTxid().equals(tx4.getId());
    AbortEvent abortEvent1 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent1));
    AbortEvent abortEvent2 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent2));
    AbortEvent abortEvent3 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent3));
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
}

Example 12 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class TaskTest method setUp.

@Before
public void setUp() throws ExecutionException, InterruptedException {
    final String stream2 = "stream2";
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final ScalingPolicy policy2 = ScalingPolicy.fixed(3);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration configuration2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy2).build();
    // region createStream
    streamStore.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, stream1, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, stream1, State.ACTIVE, null, executor).join();
    streamStore.createStream(SCOPE, stream2, configuration2, start, null, executor).join();
    streamStore.setState(SCOPE, stream2, State.ACTIVE, null, executor).join();
    // endregion
    // region scaleSegments
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStore.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStore.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment3 = new AbstractMap.SimpleEntry<>(0.0, 0.5);
    AbstractMap.SimpleEntry<Double, Double> segment4 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment5 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments1 = Arrays.asList(0, 1, 2);
    response = streamStore.startScale(SCOPE, stream2, sealedSegments1, Arrays.asList(segment3, segment4, segment5), start + 20, false, null, executor).get();
    segmentsCreated = response.getSegmentsCreated();
    streamStore.setState(SCOPE, stream2, State.SCALING, null, executor).get();
    streamStore.scaleNewSegmentsCreated(SCOPE, stream2, sealedSegments1, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStore.scaleSegmentsSealed(SCOPE, stream2, sealedSegments1.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
// endregion
}

Example 13 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class TaskTest method testTaskSweeper.

@Test
public void testTaskSweeper() throws ExecutionException, InterruptedException {
    final String deadHost = "deadHost";
    final String deadThreadId = UUID.randomUUID().toString();
    final String scope = SCOPE;
    final String stream = "streamSweeper";
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final Resource resource = new Resource(scope, stream);
    final long timestamp = System.currentTimeMillis();
    final TaskData taskData = new TaskData("createStream", "1.0", new Serializable[] { scope, stream, configuration, timestamp });
    for (int i = 0; i < 5; i++) {
        final TaggedResource taggedResource = new TaggedResource(UUID.randomUUID().toString(), resource);
        taskMetadataStore.putChild(deadHost, taggedResource).join();
    }
    final TaggedResource taggedResource = new TaggedResource(deadThreadId, resource);
    taskMetadataStore.putChild(deadHost, taggedResource).join();
    taskMetadataStore.lock(resource, taskData, deadHost, deadThreadId, null, null).join();
    TaskSweeper taskSweeper = new TaskSweeper(taskMetadataStore, HOSTNAME, executor, streamMetadataTasks);
    taskSweeper.handleFailedProcess(deadHost).get();
    Optional<TaskData> data = taskMetadataStore.getTask(resource, deadHost, deadThreadId).get();
    assertFalse(data.isPresent());
    Optional<TaggedResource> child = taskMetadataStore.getRandomChild(deadHost).get();
    assertFalse(child.isPresent());
    // ensure that the stream streamSweeper is created
    StreamConfiguration config = streamStore.getConfiguration(SCOPE, stream, null, executor).get();
    assertTrue(config.getStreamName().equals(configuration.getStreamName()));
    assertTrue(config.getScope().equals(configuration.getScope()));
    assertTrue(config.getScalingPolicy().equals(configuration.getScalingPolicy()));
}

Example 14 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class TaskTest method parallelTaskSweeperTest.

@Test
public void parallelTaskSweeperTest() throws InterruptedException, ExecutionException {
    final String deadHost = "deadHost";
    final String deadThreadId1 = UUID.randomUUID().toString();
    final String deadThreadId2 = UUID.randomUUID().toString();
    final String scope = SCOPE;
    final String stream1 = "parallelSweeper1";
    final String stream2 = "parallelSweeper2";
    final StreamConfiguration config1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    final StreamConfiguration config2 = StreamConfiguration.builder().scope(SCOPE).streamName(stream2).scalingPolicy(policy1).build();
    final Resource resource1 = new Resource(scope, stream1);
    final long timestamp1 = System.currentTimeMillis();
    final TaskData taskData1 = new TaskData("createStream", "1.0", new Serializable[] { scope, stream1, config1, timestamp1 });
    final Resource resource2 = new Resource(scope, stream2);
    final long timestamp2 = System.currentTimeMillis();
    final TaskData taskData2 = new TaskData("createStream", "1.0", new Serializable[] { scope, stream2, config2, timestamp2 });
    for (int i = 0; i < 5; i++) {
        final TaggedResource taggedResource = new TaggedResource(UUID.randomUUID().toString(), resource1);
        taskMetadataStore.putChild(deadHost, taggedResource).join();
    }
    final TaggedResource taggedResource1 = new TaggedResource(deadThreadId1, resource1);
    taskMetadataStore.putChild(deadHost, taggedResource1).join();
    final TaggedResource taggedResource2 = new TaggedResource(deadThreadId2, resource2);
    taskMetadataStore.putChild(deadHost, taggedResource2).join();
    taskMetadataStore.lock(resource1, taskData1, deadHost, deadThreadId1, null, null).join();
    taskMetadataStore.lock(resource2, taskData2, deadHost, deadThreadId2, null, null).join();
    final SweeperThread sweeperThread1 = new SweeperThread(HOSTNAME, executor, taskMetadataStore, streamMetadataTasks, deadHost);
    final SweeperThread sweeperThread2 = new SweeperThread(HOSTNAME, executor, taskMetadataStore, streamMetadataTasks, deadHost);
    sweeperThread1.start();
    sweeperThread2.start();
    sweeperThread1.getResult().join();
    sweeperThread2.getResult().join();
    Optional<TaskData> data = taskMetadataStore.getTask(resource1, deadHost, deadThreadId1).get();
    assertFalse(data.isPresent());
    data = taskMetadataStore.getTask(resource2, deadHost, deadThreadId2).get();
    assertFalse(data.isPresent());
    Optional<TaggedResource> child = taskMetadataStore.getRandomChild(deadHost).get();
    assertFalse(child.isPresent());
    // ensure that the stream streamSweeper is created
    StreamConfiguration config = streamStore.getConfiguration(SCOPE, stream1, null, executor).get();
    assertTrue(config.getStreamName().equals(stream1));
    config = streamStore.getConfiguration(SCOPE, stream2, null, executor).get();
    assertTrue(config.getStreamName().equals(stream2));
}

Example 15 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class ModelHelperTest method testEncodeStreamResponse.

@Test
public void testEncodeStreamResponse() {
    StreamConfiguration streamConfig = StreamConfiguration.builder().streamName("stream").scope("scope").scalingPolicy(ScalingPolicy.fixed(1)).build();
    StreamProperty streamProperty = encodeStreamResponse(streamConfig);
    Assert.assertEquals("scope", streamProperty.getScopeName());
    Assert.assertEquals("stream", streamProperty.getStreamName());
    Assert.assertEquals(ScalingConfig.TypeEnum.FIXED_NUM_SEGMENTS, streamProperty.getScalingPolicy().getType());
    Assert.assertEquals((Integer) 1, streamProperty.getScalingPolicy().getMinSegments());
    Assert.assertNull(streamProperty.getRetentionPolicy());
    streamConfig = StreamConfiguration.builder().streamName("stream").scope("scope").scalingPolicy(ScalingPolicy.byDataRate(100, 200, 1)).retentionPolicy(RetentionPolicy.byTime(Duration.ofDays(100L))).build();
    streamProperty = encodeStreamResponse(streamConfig);
    Assert.assertEquals(ScalingConfig.TypeEnum.BY_RATE_IN_KBYTES_PER_SEC, streamProperty.getScalingPolicy().getType());
    Assert.assertEquals((Integer) 1, streamProperty.getScalingPolicy().getMinSegments());
    Assert.assertEquals((Integer) 100, streamProperty.getScalingPolicy().getTargetRate());
    Assert.assertEquals((Integer) 200, streamProperty.getScalingPolicy().getScaleFactor());
    Assert.assertEquals(RetentionConfig.TypeEnum.LIMITED_DAYS, streamProperty.getRetentionPolicy().getType());
    Assert.assertEquals((Long) 100L, streamProperty.getRetentionPolicy().getValue());
    streamConfig = StreamConfiguration.builder().streamName("stream").scope("scope").scalingPolicy(ScalingPolicy.byEventRate(100, 200, 1)).retentionPolicy(RetentionPolicy.bySizeBytes(1234L * 1024 * 1024)).build();
    streamProperty = encodeStreamResponse(streamConfig);
    Assert.assertEquals(ScalingConfig.TypeEnum.BY_RATE_IN_EVENTS_PER_SEC, streamProperty.getScalingPolicy().getType());
    Assert.assertEquals((Integer) 1, streamProperty.getScalingPolicy().getMinSegments());
    Assert.assertEquals((Integer) 100, streamProperty.getScalingPolicy().getTargetRate());
    Assert.assertEquals((Integer) 200, streamProperty.getScalingPolicy().getScaleFactor());
    Assert.assertEquals(RetentionConfig.TypeEnum.LIMITED_SIZE_MB, streamProperty.getRetentionPolicy().getType());
    Assert.assertEquals((Long) 1234L, streamProperty.getRetentionPolicy().getValue());
}