Examples with ScalingPolicy io.pravega.client.stream.ScalingPolicy used on opensource projects

Example 6 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class ZkStreamTest method testZkDeleteScope.

@Test
public void testZkDeleteScope() throws Exception {
    // create new scope
    final StreamMetadataStore store = new ZKStreamMetadataStore(cli, executor);
    final String scopeName = "Scope1";
    store.createScope(scopeName).get();
    // Delete empty scope Scope1
    CompletableFuture<DeleteScopeStatus> deleteScopeStatus = store.deleteScope(scopeName);
    assertEquals("Delete Empty Scope", DeleteScopeStatus.Status.SUCCESS, deleteScopeStatus.get().getStatus());
    // Delete non-existent scope Scope2
    CompletableFuture<DeleteScopeStatus> deleteScopeStatus2 = store.deleteScope("Scope2");
    assertEquals("Delete non-existent Scope", DeleteScopeStatus.Status.SCOPE_NOT_FOUND, deleteScopeStatus2.get().getStatus());
    // Delete non-empty scope Scope3
    store.createScope("Scope3").get();
    final ScalingPolicy policy = ScalingPolicy.fixed(5);
    final StreamConfiguration streamConfig = StreamConfiguration.builder().scope("Scope3").streamName("Stream3").scalingPolicy(policy).build();
    store.createStream("Scope3", "Stream3", streamConfig, System.currentTimeMillis(), null, executor).get();
    store.setState("Scope3", "Stream3", State.ACTIVE, null, executor).get();
    CompletableFuture<DeleteScopeStatus> deleteScopeStatus3 = store.deleteScope("Scope3");
    assertEquals("Delete non-empty Scope", DeleteScopeStatus.Status.SCOPE_NOT_EMPTY, deleteScopeStatus3.get().getStatus());
}

Example 7 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataTasksTest method sizeBasedRetentionStreamTest.

@Test(timeout = 30000)
public void sizeBasedRetentionStreamTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final RetentionPolicy retentionPolicy = RetentionPolicy.builder().retentionType(RetentionPolicy.RetentionType.SIZE).retentionParam(100L).build();
    String streamName = "test";
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName(streamName).scalingPolicy(policy).retentionPolicy(retentionPolicy).build();
    streamStorePartialMock.createStream(SCOPE, streamName, configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, streamName, State.ACTIVE, null, executor).get();
    assertNotEquals(0, consumer.getCurrentSegments(SCOPE, streamName).get().size());
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    // region size based retention on stream cuts on epoch 0
    // region no previous streamcut
    // first retention iteration
    // streamcut1: 19 bytes(0/9,1/10)
    long recordingTime1 = System.currentTimeMillis();
    Map<Integer, Long> map1 = new HashMap<>();
    map1.put(0, 9L);
    map1.put(1, 10L);
    long size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map1, null, executor).join();
    assertEquals(size, 19);
    StreamCutRecord streamCut1 = new StreamCutRecord(recordingTime1, size, map1);
    doReturn(CompletableFuture.completedFuture(streamCut1)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), any());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime1, null, "").get();
    // verify that one streamCut is generated and added.
    List<StreamCutRecord> list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    assertTrue(list.contains(streamCut1));
    // endregion
    // region stream cut exists but latest - previous < retention.size
    // second retention iteration
    // streamcut2: 100 bytes(0/50, 1/50)
    Map<Integer, Long> map2 = new HashMap<>();
    map2.put(0, 50L);
    map2.put(1, 50L);
    long recordingTime2 = recordingTime1 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map2, null, executor).join();
    assertEquals(size, 100L);
    StreamCutRecord streamCut2 = new StreamCutRecord(recordingTime2, size, map2);
    doReturn(CompletableFuture.completedFuture(streamCut2)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime2, null, "").get();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    StreamProperty<StreamTruncationRecord> truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    // verify that two stream cut is in retention set. streamCut2 is added
    // verify that truncation did not happen
    assertTrue(list.contains(streamCut1));
    assertTrue(list.contains(streamCut2));
    assertTrue(!truncProp.isUpdating());
    // endregion
    // region latest - previous > retention.size
    // third retention iteration
    // streamcut3: 120 bytes(0/60, 1/60)
    Map<Integer, Long> map3 = new HashMap<>();
    map3.put(0, 60L);
    map3.put(1, 60L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map3, null, executor).join();
    assertEquals(size, 120L);
    long recordingTime3 = recordingTime2 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut3 = new StreamCutRecord(recordingTime3, size, map3);
    doReturn(CompletableFuture.completedFuture(streamCut3)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime3, null, "").get();
    // verify two stream cuts are in retention set. Cut 2 and 3.
    // verify that Truncation has happened.
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertTrue(!list.contains(streamCut1));
    assertTrue(list.contains(streamCut2));
    assertTrue(list.contains(streamCut3));
    assertTrue(truncProp.isUpdating());
    assertTrue(truncProp.getProperty().getStreamCut().get(0) == 9L && truncProp.getProperty().getStreamCut().get(1) == 10L);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(truncProp.isUpdating());
    // endregion
    // endregion
    // region test retention over multiple epochs
    // scale1 --> seal segments 0 and 1 and create 2 and 3. (0/70, 1/70)
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    Map<Integer, Long> sealedSegmentsWithSize = new HashMap<>();
    sealedSegmentsWithSize.put(0, 70L);
    sealedSegmentsWithSize.put(1, 70L);
    scale(SCOPE, streamName, sealedSegmentsWithSize, newRanges);
    // region latest streamcut on new epoch but latest (newepoch) - previous (oldepoch) < retention.size
    // 4th retention iteration
    // streamcut4: (2/29, 3/30)
    Map<Integer, Long> map4 = new HashMap<>();
    map4.put(2, 29L);
    map4.put(3, 30L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map4, null, executor).join();
    assertEquals(size, 199L);
    long recordingTime4 = recordingTime3 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut4 = new StreamCutRecord(recordingTime4, size, map4);
    doReturn(CompletableFuture.completedFuture(streamCut4)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime4, null, "").get();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(list.contains(streamCut1));
    assertTrue(list.contains(streamCut2));
    assertTrue(list.contains(streamCut3));
    assertTrue(list.contains(streamCut4));
    assertFalse(truncProp.isUpdating());
    // endregion
    // region latest streamcut on new epoch but latest (newepoch) - previous (oldepoch) > retention.size
    // 5th retention iteration
    // streamcut5: 221 bytes(2/41, 3/40)
    Map<Integer, Long> map5 = new HashMap<>();
    map5.put(2, 41L);
    map5.put(3, 40L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map5, null, executor).join();
    assertEquals(size, 221L);
    long recordingTime5 = recordingTime4 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut5 = new StreamCutRecord(recordingTime5, size, map5);
    doReturn(CompletableFuture.completedFuture(streamCut5)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime5, null, "").get();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(list.contains(streamCut1));
    assertFalse(list.contains(streamCut2));
    assertFalse(list.contains(streamCut3));
    assertTrue(list.contains(streamCut4));
    assertTrue(list.contains(streamCut5));
    assertTrue(truncProp.isUpdating());
    assertTrue(truncProp.getProperty().getStreamCut().get(0) == 60L && truncProp.getProperty().getStreamCut().get(1) == 60L);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(truncProp.isUpdating());
    // endregion
    // region test retention with external manual truncation
    // scale2 -->  split segment 2 to 4 and 5. Sealed size for segment 2 = 50
    newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.25));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.25, 0.5));
    sealedSegmentsWithSize = new HashMap<>();
    sealedSegmentsWithSize.put(2, 50L);
    scale(SCOPE, streamName, sealedSegmentsWithSize, newRanges);
    // region add streamcut on new epoch such that latest - oldest < retention.size
    // streamcut6: 290 bytes (3/40, 4/30, 5/30)
    // verify no new truncation happens..
    Map<Integer, Long> map6 = new HashMap<>();
    map6.put(3, 40L);
    map6.put(4, 30L);
    map6.put(5, 30L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map6, null, executor).join();
    assertEquals(size, 290L);
    long recordingTime6 = recordingTime5 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut6 = new StreamCutRecord(recordingTime6, size, map6);
    doReturn(CompletableFuture.completedFuture(streamCut6)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime6, null, "").get();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(list.contains(streamCut1));
    assertFalse(list.contains(streamCut2));
    assertFalse(list.contains(streamCut3));
    assertTrue(list.contains(streamCut4));
    assertTrue(list.contains(streamCut5));
    assertTrue(list.contains(streamCut6));
    assertFalse(truncProp.isUpdating());
    // endregion
    // truncate on manual streamcutManual: (1/65, 4/10, 5/10)
    Map<Integer, Long> streamCutManual = new HashMap<>();
    streamCutManual.put(1, 65L);
    streamCutManual.put(4, 10L);
    streamCutManual.put(5, 10L);
    CompletableFuture<UpdateStreamStatus.Status> future = streamMetadataTasks.truncateStream(SCOPE, streamName, streamCutManual, null);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    assertTrue(Futures.await(future));
    assertEquals(future.join(), UpdateStreamStatus.Status.SUCCESS);
    // streamcut7: 340 bytes (3/50, 4/50, 5/50)
    Map<Integer, Long> map7 = new HashMap<>();
    map7.put(3, 50L);
    map7.put(4, 50L);
    map7.put(5, 50L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map7, null, executor).join();
    assertEquals(size, 340L);
    long recordingTime7 = recordingTime6 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut7 = new StreamCutRecord(recordingTime7, size, map7);
    doReturn(CompletableFuture.completedFuture(streamCut7)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    // verify no new truncation.. streamcut5 should be chosen but discarded because it is not strictly-ahead-of-truncationRecord
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime7, null, "").join();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(list.contains(streamCut1));
    assertFalse(list.contains(streamCut2));
    assertFalse(list.contains(streamCut3));
    assertTrue(list.contains(streamCut4));
    assertTrue(list.contains(streamCut5));
    assertTrue(list.contains(streamCut6));
    assertTrue(list.contains(streamCut7));
    assertFalse(truncProp.isUpdating());
    // streamcut8: 400 bytes (3/70, 4/70, 5/70)
    Map<Integer, Long> map8 = new HashMap<>();
    map8.put(3, 70L);
    map8.put(4, 70L);
    map8.put(5, 70L);
    size = streamStorePartialMock.getSizeTillStreamCut(SCOPE, streamName, map8, null, executor).join();
    assertEquals(size, 400L);
    long recordingTime8 = recordingTime7 + Duration.ofMinutes(Config.MINIMUM_RETENTION_FREQUENCY_IN_MINUTES).toMillis() + 1;
    StreamCutRecord streamCut8 = new StreamCutRecord(recordingTime8, size, map8);
    doReturn(CompletableFuture.completedFuture(streamCut8)).when(streamMetadataTasks).generateStreamCut(anyString(), anyString(), any(), anyString());
    streamMetadataTasks.retention(SCOPE, streamName, retentionPolicy, recordingTime8, null, "").get();
    list = streamStorePartialMock.getStreamCutsFromRetentionSet(SCOPE, streamName, null, executor).get();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    // verify truncation happens at streamcut6
    assertFalse(list.contains(streamCut1));
    assertFalse(list.contains(streamCut2));
    assertFalse(list.contains(streamCut3));
    assertFalse(list.contains(streamCut4));
    assertFalse(list.contains(streamCut5));
    assertFalse(list.contains(streamCut6));
    assertTrue(list.contains(streamCut7));
    assertTrue(truncProp.isUpdating());
    assertTrue(truncProp.getProperty().getStreamCut().get(3) == 40L && truncProp.getProperty().getStreamCut().get(4) == 30L && truncProp.getProperty().getStreamCut().get(5) == 30L);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, streamName, true, null, executor).get();
    assertFalse(truncProp.isUpdating());
// endregion
// endregion
}

Example 8 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataTasksTest method eventWriterInitializationTest.

@Test
public void eventWriterInitializationTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(1);
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName("test").scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    AssertExtensions.assertThrows("", () -> streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(0), Arrays.asList(), 30, null).get(), e -> e instanceof TaskExceptions.ProcessingDisabledException);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, executor));
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.0, 0.5));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 1.0));
    ScaleResponse scaleOpResult = streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(0), newRanges, 30, null).get();
    assertEquals(ScaleStreamStatus.STARTED, scaleOpResult.getStatus());
    Controller.ScaleStatusResponse scaleStatusResult = streamMetadataTasks.checkScale(SCOPE, "UNKNOWN", 0, null).get();
    assertEquals(Controller.ScaleStatusResponse.ScaleStatus.INVALID_INPUT, scaleStatusResult.getStatus());
    scaleStatusResult = streamMetadataTasks.checkScale("UNKNOWN", "test", 0, null).get();
    assertEquals(Controller.ScaleStatusResponse.ScaleStatus.INVALID_INPUT, scaleStatusResult.getStatus());
    scaleStatusResult = streamMetadataTasks.checkScale(SCOPE, "test", 5, null).get();
    assertEquals(Controller.ScaleStatusResponse.ScaleStatus.INVALID_INPUT, scaleStatusResult.getStatus());
}

Example 9 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamMetadataTasksTest method truncateStreamTest.

@Test(timeout = 30000)
public void truncateStreamTest() throws Exception {
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration = StreamConfiguration.builder().scope(SCOPE).streamName("test").scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, "test", configuration, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    assertNotEquals(0, consumer.getCurrentSegments(SCOPE, "test").get().size());
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    List<AbstractMap.SimpleEntry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 0.75));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.75, 1.0));
    ScaleResponse scaleOpResult = streamMetadataTasks.manualScale(SCOPE, "test", Collections.singletonList(1), newRanges, 30, null).get();
    assertTrue(scaleOpResult.getStatus().equals(ScaleStreamStatus.STARTED));
    ScaleOperationTask scaleTask = new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor);
    assertTrue(Futures.await(scaleTask.execute((ScaleOpEvent) requestEventWriter.eventQueue.take())));
    // start truncation
    StreamProperty<StreamTruncationRecord> truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).join();
    assertFalse(truncProp.isUpdating());
    // 1. happy day test
    // update.. should succeed
    Map<Integer, Long> streamCut = new HashMap<>();
    streamCut.put(0, 1L);
    streamCut.put(1, 11L);
    CompletableFuture<UpdateStreamStatus.Status> truncateFuture = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut, null);
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    assertEquals(UpdateStreamStatus.Status.SUCCESS, truncateFuture.join());
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).join();
    assertTrue(truncProp.getProperty().getStreamCut().equals(streamCut));
    assertTrue(truncProp.getProperty().getStreamCut().equals(streamCut));
    // 2. change state to scaling
    streamStorePartialMock.setState(SCOPE, "test", State.SCALING, null, executor).get();
    // call update should fail without posting the event
    Map<Integer, Long> streamCut2 = new HashMap<>();
    streamCut2.put(0, 1L);
    streamCut2.put(2, 1L);
    streamCut2.put(3, 1L);
    streamMetadataTasks.truncateStream(SCOPE, "test", streamCut2, null);
    AtomicBoolean loop = new AtomicBoolean(false);
    Futures.loop(() -> !loop.get(), () -> streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).thenApply(StreamProperty::isUpdating).thenAccept(loop::set), executor).join();
    // event posted, first step performed. now pick the event for processing
    TruncateStreamTask truncateStreamTask = new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor);
    TruncateStreamEvent taken = (TruncateStreamEvent) requestEventWriter.eventQueue.take();
    AssertExtensions.assertThrows("", truncateStreamTask.execute(taken), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    streamStorePartialMock.setState(SCOPE, "test", State.ACTIVE, null, executor).get();
    // now with state = active, process the same event. it should succeed now.
    assertTrue(Futures.await(truncateStreamTask.execute(taken)));
    // 3. multiple back to back updates.
    Map<Integer, Long> streamCut3 = new HashMap<>();
    streamCut3.put(0, 12L);
    streamCut3.put(2, 12L);
    streamCut3.put(3, 12L);
    CompletableFuture<UpdateStreamStatus.Status> truncateOp1 = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut3, null);
    // ensure that previous updatestream has posted the event and set status to updating,
    // only then call second updateStream
    AtomicBoolean loop2 = new AtomicBoolean(false);
    Futures.loop(() -> !loop2.get(), () -> streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).thenApply(StreamProperty::isUpdating).thenAccept(loop2::set), executor).join();
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).join();
    assertTrue(truncProp.getProperty().getStreamCut().equals(streamCut3) && truncProp.isUpdating());
    // post the second update request. This should fail here itself as previous one has started.
    Map<Integer, Long> streamCut4 = new HashMap<>();
    streamCut4.put(0, 14L);
    streamCut4.put(2, 14L);
    streamCut4.put(3, 14L);
    CompletableFuture<UpdateStreamStatus.Status> truncateOpFuture2 = streamMetadataTasks.truncateStream(SCOPE, "test", streamCut4, null);
    assertEquals(UpdateStreamStatus.Status.FAILURE, truncateOpFuture2.join());
    // process event
    assertTrue(Futures.await(processEvent(requestEventWriter)));
    // verify that first request for update also completes with success.
    assertEquals(UpdateStreamStatus.Status.SUCCESS, truncateOp1.join());
    truncProp = streamStorePartialMock.getTruncationProperty(SCOPE, "test", true, null, executor).join();
    assertTrue(truncProp.getProperty().getStreamCut().equals(streamCut3) && !truncProp.isUpdating());
}

Example 10 with ScalingPolicy

use of io.pravega.client.stream.ScalingPolicy in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method failOverTests.

@Test
public void failOverTests() throws CheckpointStoreException, InterruptedException {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    EventStreamReader<CommitEvent> commitReader = commitWriter.getReader();
    EventStreamReader<AbortEvent> abortReader = abortWriter.getReader();
    consumer = new ControllerService(streamStore, hostStore, streamMetadataTasks, txnTasks, segmentHelperMock, executor, null);
    // Create test scope and stream.
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scope(SCOPE).streamName(STREAM).scalingPolicy(policy1).build();
    Assert.assertEquals(Controller.CreateScopeStatus.Status.SUCCESS, consumer.createScope(SCOPE).join().getStatus());
    Assert.assertEquals(Controller.CreateStreamStatus.Status.SUCCESS, streamMetadataTasks.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis()).join());
    // Set up txn task for creating transactions from a failedHost.
    StreamTransactionMetadataTasks failedTxnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "failedHost", connectionFactory, false, "");
    failedTxnTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>());
    // Create 3 transactions from failedHost.
    VersionedTransactionData tx1 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx2 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    VersionedTransactionData tx3 = failedTxnTasks.createTxn(SCOPE, STREAM, 10000, 10000, null).join().getKey();
    // Ping another txn from failedHost.
    UUID txnId = UUID.randomUUID();
    streamStore.createTransaction(SCOPE, STREAM, txnId, 10000, 30000, 30000, null, executor).join();
    PingTxnStatus pingStatus = failedTxnTasks.pingTxn(SCOPE, STREAM, txnId, 10000, null).join();
    VersionedTransactionData tx4 = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    // Validate versions of all txn
    Assert.assertEquals(0, tx1.getVersion());
    Assert.assertEquals(0, tx2.getVersion());
    Assert.assertEquals(0, tx3.getVersion());
    Assert.assertEquals(1, tx4.getVersion());
    Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
    // Validate the txn index.
    Assert.assertEquals(1, streamStore.listHostsOwningTxn().join().size());
    // Change state of one txn to COMMITTING.
    TxnStatus txnStatus2 = streamStore.sealTransaction(SCOPE, STREAM, tx2.getId(), true, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.COMMITTING, txnStatus2);
    // Change state of another txn to ABORTING.
    TxnStatus txnStatus3 = streamStore.sealTransaction(SCOPE, STREAM, tx3.getId(), false, Optional.empty(), null, executor).thenApply(AbstractMap.SimpleEntry::getKey).join();
    Assert.assertEquals(TxnStatus.ABORTING, txnStatus3);
    // Create transaction tasks for sweeping txns from failedHost.
    txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelperMock, executor, "host", connectionFactory, false, "");
    TxnSweeper txnSweeper = new TxnSweeper(streamStore, txnTasks, 100, executor);
    // Before initializing, txnSweeper.sweepFailedHosts would throw an error
    AssertExtensions.assertThrows("IllegalStateException before initialization", txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")), ex -> ex instanceof IllegalStateException);
    // Initialize stream writers.
    txnTasks.initializeStreamWriters("commitStream", commitWriter, "abortStream", abortWriter);
    // Validate that txnTasks is ready.
    assertTrue(txnTasks.isReady());
    // Sweep txns that were being managed by failedHost.
    txnSweeper.sweepFailedProcesses(() -> Collections.singleton("host")).join();
    // Validate that sweeping completes correctly.
    Assert.assertEquals(0, streamStore.listHostsOwningTxn().join().size());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.COMMITTING, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    Assert.assertEquals(TxnStatus.ABORTING, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
    // Create commit and abort event processors.
    ConnectionFactory connectionFactory = Mockito.mock(ConnectionFactory.class);
    BlockingQueue<CommitEvent> processedCommitEvents = new LinkedBlockingQueue<>();
    BlockingQueue<AbortEvent> processedAbortEvents = new LinkedBlockingQueue<>();
    createEventProcessor("commitRG", "commitStream", commitReader, commitWriter, () -> new CommitEventProcessor(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedCommitEvents));
    createEventProcessor("abortRG", "abortStream", abortReader, abortWriter, () -> new ConcurrentEventProcessor<>(new AbortRequestHandler(streamStore, streamMetadataTasks, hostStore, executor, segmentHelperMock, connectionFactory, processedAbortEvents), executor));
    // Wait until the commit event is processed and ensure that the txn state is COMMITTED.
    CommitEvent commitEvent = processedCommitEvents.take();
    assertEquals(tx2.getId(), commitEvent.getTxid());
    assertEquals(TxnStatus.COMMITTED, streamStore.transactionStatus(SCOPE, STREAM, tx2.getId(), null, executor).join());
    // Wait until 3 abort events are processed and ensure that the txn state is ABORTED.
    Predicate<AbortEvent> predicate = event -> event.getTxid().equals(tx1.getId()) || event.getTxid().equals(tx3.getId()) || event.getTxid().equals(tx4.getId());
    AbortEvent abortEvent1 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent1));
    AbortEvent abortEvent2 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent2));
    AbortEvent abortEvent3 = processedAbortEvents.take();
    assertTrue(predicate.test(abortEvent3));
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx1.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx3.getId(), null, executor).join());
    assertEquals(TxnStatus.ABORTED, streamStore.transactionStatus(SCOPE, STREAM, tx4.getId(), null, executor).join());
}