Examples with StreamConfiguration io.pravega.client.stream.StreamConfiguration used on opensource projects

Example 56 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamMetadataTasksTest method manualScaleTest.

@Test(timeout = 30000)
public void manualScaleTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(1);
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName("test").scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    ScaleResponse scaleOpResult = streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(0), newRanges, 30, null).get();
    assertEquals(ScaleStreamStatus.STARTED, scaleOpResult.getStatus());
    OperationContext context = streamStorePartialMock.createContext(SCOPE, "test");
    assertEquals(streamStorePartialMock.getState(SCOPE, "test", false, context, executor).get(), State.ACTIVE);
    // Now when startScale runs even after that we should get the state as active.
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, "test", Collections.singletonList(0), newRanges, 30, true, null, executor).get();
    assertEquals(response.getActiveEpoch(), 0);
    assertEquals(streamStorePartialMock.getState(SCOPE, "test", true, context, executor).get(), State.ACTIVE);
    AssertExtensions.assertThrows("", () -> streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, "test", Collections.singletonList(0), response.getSegmentsCreated(), response.getActiveEpoch(), 30, context, executor).get(), ex -> Exceptions.unwrap(ex) instanceof StoreException.IllegalStateException);
    List<Segment> segments = streamMetadataTasks.startScale((ScaleOpEvent) requestEventWriter.getEventQueue().take(), true, context, "").get();
    assertTrue(segments.stream().anyMatch(x -> x.getNumber() == 1 && x.getKeyStart() == 0.0 && x.getKeyEnd() == 0.5));
    assertTrue(segments.stream().anyMatch(x -> x.getNumber() == 2 && x.getKeyStart() == 0.5 && x.getKeyEnd() == 1.0));
}

Example 57 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore(1, executor);
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    doReturn(CompletableFuture.completedFuture(false)).when(streamStorePartialMock).isTransactionOngoing(anyString(), anyString(), any(), // mock only isTransactionOngoing call.
    any());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, hostStore, taskMetadataStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key"));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, hostStore, segmentHelperMock, executor, "host", connectionFactory, authEnabled, "key");
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new SealStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), executor);
    consumer = new ControllerService(streamStorePartialMock, hostStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(stream1).scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE).join();
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Integer> sealedSegments = Collections.singletonList(1);
    StartScaleResponse response = streamStorePartialMock.startScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, false, null, executor).get();
    List<Segment> segmentsCreated = response.getSegmentsCreated();
    streamStorePartialMock.setState(SCOPE, stream1, State.SCALING, null, executor).get();
    streamStorePartialMock.scaleNewSegmentsCreated(SCOPE, stream1, sealedSegments, segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), segmentsCreated, response.getActiveEpoch(), start + 20, null, executor).get();
}

Example 58 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamMetadataTasksTest method retentionPolicyUpdateTest.

@Test(timeout = 30000)
public void retentionPolicyUpdateTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    String stream = "test";
    final StreamConfiguration noRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).build();
    // add stream without retention policy
    streamMetadataTasks.createStreamBody(SCOPE, stream, noRetentionConfig, System.currentTimeMillis()).join();
    String scopedStreamName = String.format("%s/%s", SCOPE, stream);
    // verify that stream is not added to bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    UpdateStreamTask task = new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    final RetentionPolicy retentionPolicy = RetentionPolicy.builder().retentionType(RetentionPolicy.RetentionType.TIME).retentionParam(Duration.ofMinutes(60).toMillis()).build();
    final StreamConfiguration withRetentionConfig = StreamConfiguration.builder().scope(SCOPE).streamName(stream).scalingPolicy(policy).retentionPolicy(retentionPolicy).build();
    // now update stream with a retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, withRetentionConfig, null, executor).join();
    UpdateStreamEvent update = new UpdateStreamEvent(SCOPE, stream);
    task.execute(update).join();
    // verify that bucket has the stream.
    assertTrue(streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
    // update stream such that stream is updated with null retention policy
    streamStorePartialMock.startUpdateConfiguration(SCOPE, stream, noRetentionConfig, null, executor).join();
    task.execute(update).join();
    // verify that the stream is no longer present in the bucket
    assertTrue(!streamStorePartialMock.getStreamsForBucket(0, executor).join().contains(scopedStreamName));
}

Example 59 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method commitAbortTests.

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void commitAbortTests() {
    // Create mock writer objects.
    final List<CompletableFuture<Void>> commitWriterResponses = getWriteResultSequence(5);
    final List<CompletableFuture<Void>> abortWriterResponses = getWriteResultSequence(5);
    EventStreamWriter<CommitEvent> commitWriter = Mockito.mock(EventStreamWriter.class);
    Mockito.when(commitWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(commitWriterResponses));
    EventStreamWriter<AbortEvent> abortWriter = Mockito.mock(EventStreamWriter.class);
    Mockito.when(abortWriter.writeEvent(anyString(), any())).thenAnswer(new SequenceAnswer<>(abortWriterResponses));
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Create ControllerService.
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Create stream and scope
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
    // Create 2 transactions
    final long lease = 5000;
    final long scaleGracePeriod = 10000;
    VersionedTransactionData txData1 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    VersionedTransactionData txData2 = txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null).join().getKey();
    // Commit the first one
    TxnStatus status = txnTasks.commitTxn(SCOPE, STREAM, txData1.getId(), null).join();
    Assert.assertEquals(TxnStatus.COMMITTING, status);
    // Abort the second one
    status = txnTasks.abortTxn(SCOPE, STREAM, txData2.getId(), txData2.getVersion(), null).join();
    Assert.assertEquals(TxnStatus.ABORTING, status);
}

Example 60 with StreamConfiguration

use of io.pravega.client.stream.StreamConfiguration in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method partialTxnCreationTest.

@Test(timeout = 10000)
public void partialTxnCreationTest() {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, SegmentHelperMock.getFailingSegmentHelperMock(), executor, "host", connectionFactory, this.authEnabled, "secret");
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Create ControllerService.
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    // Create stream and scope
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, 0).join());
    // Create partial transaction
    final long lease = 10000;
    final long scaleGracePeriod = 10000;
    AssertExtensions.assertThrows("Transaction creation fails, although a new txn id gets added to the store", txnTasks.createTxn(SCOPE, STREAM, lease, scaleGracePeriod, null), e -> e instanceof RuntimeException);
    // Ensure that exactly one transaction is active on the stream.
    Set<UUID> txns = streamStore.getActiveTxns(SCOPE, STREAM, null, executor).join().keySet();
    assertEquals(1, txns.size());
    // Ensure that transaction state is OPEN.
    UUID txn1 = txns.stream().findFirst().get();
    assertEquals(TxnStatus.OPEN, streamStore.transactionStatus(SCOPE, STREAM, txn1, null, executor).join());
    // Ensure that timeout service knows about the transaction.
    assertTrue(txnTasks.getTimeoutService().containsTxn(SCOPE, STREAM, txn1));
}