Examples with StreamMetadataStore io.pravega.controller.store.stream.StreamMetadataStore used on opensource projects

Example 31 with StreamMetadataStore

use of io.pravega.controller.store.stream.StreamMetadataStore in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    StreamMetadataStore streamStore = getStore();
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    ImmutableMap<BucketStore.ServiceType, Integer> map = ImmutableMap.of(BucketStore.ServiceType.RetentionService, 1, BucketStore.ServiceType.WatermarkingService, 1);
    bucketStore = StreamStoreFactory.createInMemoryBucketStore(map);
    kvtStore = spy(getKvtStore());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper));
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300));
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStorePartialMock, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStorePartialMock, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
    commitWriter = new EventStreamWriterMock<>();
    abortWriter = new EventStreamWriterMock<>();
    streamTransactionMetadataTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE, null, executor).join();
    // stream1
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Long> sealedSegments = Collections.singletonList(1L);
    VersionedMetadata<EpochTransitionRecord> response = streamStorePartialMock.submitScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, null, null, executor).get();
    VersionedMetadata<State> state = streamStorePartialMock.getVersionedState(SCOPE, stream1, null, executor).join();
    state = streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.SCALING, state, null, executor).join();
    streamStorePartialMock.startScale(SCOPE, stream1, false, response, state, null, executor).join();
    streamStorePartialMock.scaleCreateNewEpochs(SCOPE, stream1, response, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).get();
    streamStorePartialMock.completeScale(SCOPE, stream1, response, null, executor).join();
    streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.ACTIVE, state, null, executor).get();
    // stream2
    streamStorePartialMock.createStream(SCOPE, stream2, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    streamStorePartialMock.createStream(SCOPE, stream3, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
}

Example 32 with StreamMetadataStore

use of io.pravega.controller.store.stream.StreamMetadataStore in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method writerRoutingKeyTest.

@Test(timeout = 10000)
public void writerRoutingKeyTest() throws InterruptedException {
    StreamMetadataStore streamStoreMock = StreamStoreFactory.createZKStore(zkClient, executor);
    txnTasks = new StreamTransactionMetadataTasks(streamStoreMock, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(this.authEnabled, "secret", 300));
    streamStore.createScope(SCOPE, null, executor).join();
    streamStore.createStream(SCOPE, STREAM, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), 1L, null, executor).join();
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    TestEventStreamWriter<CommitEvent> commitWriter = new TestEventStreamWriter<>();
    TestEventStreamWriter<AbortEvent> abortWriter = new TestEventStreamWriter<>();
    txnTasks.initializeStreamWriters(commitWriter, abortWriter);
    UUID txnId = UUID.randomUUID();
    txnTasks.writeAbortEvent(SCOPE, STREAM, 0, txnId, TxnStatus.ABORTING, 0L).join();
    Pair<String, AbortEvent> request = abortWriter.requestsReceived.take();
    assertEquals(request.getKey(), request.getValue().getKey());
    txnTasks.writeAbortEvent(new AbortEvent(SCOPE, STREAM, 0, txnId, 10L)).join();
    Pair<String, AbortEvent> request2 = abortWriter.requestsReceived.take();
    assertEquals(request2.getKey(), request2.getValue().getKey());
    // verify that both use the same key
    assertEquals(request.getKey(), request2.getKey());
    txnTasks.writeCommitEvent(SCOPE, STREAM, 0, txnId, TxnStatus.COMMITTING, 0L).join();
    Pair<String, CommitEvent> request3 = commitWriter.requestsReceived.take();
    assertEquals(request3.getKey(), request3.getValue().getKey());
    txnTasks.writeCommitEvent(new CommitEvent(SCOPE, STREAM, 0)).join();
    Pair<String, CommitEvent> request4 = commitWriter.requestsReceived.take();
    assertEquals(request4.getKey(), request4.getValue().getKey());
    // verify that both use the same key
    assertEquals(request3.getKey(), request4.getKey());
}

Example 33 with StreamMetadataStore

use of io.pravega.controller.store.stream.StreamMetadataStore in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method txnPingTest.

@Test(timeout = 10000)
public void txnPingTest() throws Exception {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    StreamMetadataStore streamStoreMock = spy(StreamStoreFactory.createZKStore(zkClient, executor));
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStoreMock, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(this.authEnabled, "secret", 300));
    txnTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    // Create stream and scope
    streamStoreMock.createScope(SCOPE, null, executor).join();
    streamStoreMock.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis(), null, executor).join();
    streamStoreMock.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    // Verify Ping transaction on committing transaction.
    Pair<VersionedTransactionData, List<StreamSegmentRecord>> txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 0L).join();
    UUID txnId = txn.getKey().getId();
    txnTasks.commitTxn(SCOPE, STREAM, txnId, 0L).join();
    assertEquals(PingTxnStatus.Status.COMMITTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // complete commit of transaction.
    streamStoreMock.startCommitTransactions(SCOPE, STREAM, 100, null, executor).join();
    val record = streamStoreMock.getVersionedCommittingTransactionsRecord(SCOPE, STREAM, null, executor).join();
    streamStoreMock.completeCommitTransactions(SCOPE, STREAM, record, null, executor, Collections.emptyMap()).join();
    // verify that transaction is removed from active txn
    AssertExtensions.assertFutureThrows("Fetching Active Txn record should throw DNF", streamStoreMock.getTransactionData(SCOPE, STREAM, txnId, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
    assertEquals(PingTxnStatus.Status.COMMITTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // Verify Ping transaction on an aborting transaction.
    txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 1024 * 1024L).join();
    txnId = txn.getKey().getId();
    txnTasks.abortTxn(SCOPE, STREAM, txnId, null, 0L).join();
    assertEquals(PingTxnStatus.Status.ABORTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // now complete abort so that the transaction is removed from active txn and added to completed txn.
    streamStoreMock.abortTransaction(SCOPE, STREAM, txnId, null, executor).join();
    AssertExtensions.assertFutureThrows("Fetching Active Txn record should throw DNF", streamStoreMock.getTransactionData(SCOPE, STREAM, txnId, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
    assertEquals(PingTxnStatus.Status.ABORTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // try with a non existent transaction id
    assertEquals(PingTxnStatus.Status.UNKNOWN, txnTasks.pingTxn(SCOPE, STREAM, UUID.randomUUID(), 10000L, 0L).join().getStatus());
    // Verify max execution time.
    txnTasks.setMaxExecutionTime(1L);
    txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 1024 * 1024L).join();
    UUID tid = txn.getKey().getId();
    AssertExtensions.assertEventuallyEquals(PingTxnStatus.Status.MAX_EXECUTION_TIME_EXCEEDED, () -> txnTasks.pingTxn(SCOPE, STREAM, tid, 10000L, 0L).join().getStatus(), 10000L);
    txnTasks.setMaxExecutionTime(Duration.ofDays(Config.MAX_TXN_EXECUTION_TIMEBOUND_DAYS).toMillis());
}

Example 34 with StreamMetadataStore

use of io.pravega.controller.store.stream.StreamMetadataStore in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method txnCreationTest.

@Test(timeout = 10000)
public void txnCreationTest() {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    StreamMetadataStore streamStoreMock = spy(StreamStoreFactory.createZKStore(zkClient, executor));
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStoreMock, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(this.authEnabled, "secret", 600));
    txnTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    // Create stream and scope
    streamStoreMock.createScope(SCOPE, null, executor).join();
    streamStoreMock.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis(), null, executor).join();
    streamStoreMock.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    // mock streamMetadataStore.generateTxnId should throw excecption first time.
    // Note: it should be retried.
    // now the id should have been generated
    doAnswer(new Answer<CompletableFuture<UUID>>() {

        AtomicInteger count = new AtomicInteger(0);

        @Override
        public CompletableFuture<UUID> answer(InvocationOnMock invocation) throws Throwable {
            // first time throw exception.
            if (count.getAndIncrement() == 0) {
                return Futures.failedFuture(StoreException.create(StoreException.Type.WRITE_CONFLICT, "write conflict on counter update"));
            }
            // subsequent times call origin method
            @SuppressWarnings("unchecked") CompletableFuture<UUID> future = (CompletableFuture<UUID>) invocation.callRealMethod();
            return future;
        }
    }).when(streamStoreMock).generateTransactionId(eq(SCOPE), eq(STREAM), any(), any());
    doAnswer(new Answer<CompletableFuture<VersionedTransactionData>>() {

        AtomicInteger count = new AtomicInteger(0);

        @Override
        public CompletableFuture<VersionedTransactionData> answer(InvocationOnMock invocation) throws Throwable {
            // first time throw exception.
            if (count.getAndIncrement() == 0) {
                return Futures.failedFuture(StoreException.create(StoreException.Type.DATA_NOT_FOUND, "Epoch not found"));
            }
            // subsequent times call origin method
            @SuppressWarnings("unchecked") CompletableFuture<VersionedTransactionData> future = (CompletableFuture<VersionedTransactionData>) invocation.callRealMethod();
            return future;
        }
    }).when(streamStoreMock).createTransaction(any(), any(), any(), anyLong(), anyLong(), any(), any());
    Pair<VersionedTransactionData, List<StreamSegmentRecord>> txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 1024 * 1024L).join();
    // verify that generate transaction id is called 3 times
    verify(streamStoreMock, times(3)).generateTransactionId(any(), any(), any(), any());
    // verify that create transaction is called 2 times
    verify(streamStoreMock, times(2)).createTransaction(any(), any(), any(), anyLong(), anyLong(), any(), any());
    // verify that the txn id that is generated is of type ""
    UUID txnId = txn.getKey().getId();
    assertEquals(0, (int) (txnId.getMostSignificantBits() >> 32));
    assertEquals(2, txnId.getLeastSignificantBits());
}

Example 35 with StreamMetadataStore

use of io.pravega.controller.store.stream.StreamMetadataStore in project pravega by pravega.

the class TimeoutServiceTest method testPingOwnershipTransfer.

@Test(timeout = 30000)
public void testPingOwnershipTransfer() throws Exception {
    StreamMetadataStore streamStore2 = getStore();
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createStore(PRAVEGA_ZK_CURATOR_RESOURCE.storeClient, executor);
    SegmentHelper helperMock = SegmentHelperMock.getSegmentHelperMock();
    @Cleanup StreamMetadataTasks streamMetadataTasks2 = new StreamMetadataTasks(streamStore2, bucketStore, taskMetadataStore, helperMock, executor, "2", GrpcAuthHelper.getDisabledAuthHelper());
    @Cleanup StreamTransactionMetadataTasks streamTransactionMetadataTasks2 = new StreamTransactionMetadataTasks(streamStore2, helperMock, executor, "2", TimeoutServiceConfig.defaultConfig(), new LinkedBlockingQueue<>(5), GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks2.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    // Create TimeoutService
    TimerWheelTimeoutService timeoutService2 = (TimerWheelTimeoutService) streamTransactionMetadataTasks2.getTimeoutService();
    ControllerService controllerService2 = new ControllerService(kvtStore, kvtMetadataTasks, streamStore2, bucketStore, streamMetadataTasks2, streamTransactionMetadataTasks2, helperMock, executor, null, requestTracker);
    UUID txnId = controllerService.createTransaction(SCOPE, STREAM, LEASE, 9L).thenApply(x -> x.getKey()).join();
    VersionedTransactionData txnData = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    Assert.assertEquals(txnData.getVersion(), getVersion(0));
    Optional<Throwable> result = timeoutService.getTaskCompletionQueue().poll((long) (0.75 * LEASE), TimeUnit.MILLISECONDS);
    Assert.assertNull(result);
    TxnState txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.OPEN, txnState.getState());
    // increasing lease -> total effective lease = 3 * LEASE
    PingTxnStatus pingStatus = controllerService2.pingTransaction(SCOPE, STREAM, txnId, 2 * LEASE, 9L).join();
    Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
    txnData = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    Assert.assertEquals(txnData.getVersion(), getVersion(1));
    // timeoutService1 should believe that LEASE has expired and should get non empty completion tasks
    result = timeoutService.getTaskCompletionQueue().poll((long) (1.3 * LEASE + RETRY_DELAY), TimeUnit.MILLISECONDS);
    Assert.assertNotNull(result);
    // the txn may have been attempted to be aborted by timeoutService1 but would have failed. So txn to remain open
    txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.OPEN, txnState.getState());
    // timeoutService2 should continue to wait on lease expiry and should get empty completion tasks
    result = timeoutService2.getTaskCompletionQueue().poll(0L, TimeUnit.MILLISECONDS);
    Assert.assertNull(result);
    result = timeoutService2.getTaskCompletionQueue().poll(2 * LEASE + RETRY_DELAY, TimeUnit.MILLISECONDS);
    Assert.assertNotNull(result);
    // now txn should have moved to aborting because timeoutservice2 has initiated abort
    txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.ABORTING, txnState.getState());
}