Examples with AbstractStreamMetadataStore io.pravega.controller.store.stream.AbstractStreamMetadataStore used on opensource projects

Example 1 with AbstractStreamMetadataStore

use of io.pravega.controller.store.stream.AbstractStreamMetadataStore in project pravega by pravega.

the class ControllerEventProcessorTest method setUp.

@Before
public void setUp() throws Exception {
    executor = ExecutorServiceHelpers.newScheduledThreadPool(10, "test");
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new RetryOneTime(2000));
    zkClient.start();
    streamStore = createStore();
    bucketStore = StreamStoreFactory.createZKBucketStore(PRAVEGA_ZK_CURATOR_RESOURCE.client, executor);
    hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    eventHelperMock = EventHelperMock.getEventHelperMock(executor, "1", ((AbstractStreamMetadataStore) this.streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, TaskStoreFactory.createInMemoryStore(executor), segmentHelperMock, executor, "1", GrpcAuthHelper.getDisabledAuthHelper(), eventHelperMock);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelperMock, executor, "host", GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    // region createStream
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStore.createScope(SCOPE, null, executor).join();
    long start = System.currentTimeMillis();
    streamStore.createStream(SCOPE, STREAM, configuration1, start, null, executor).join();
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
// endregion
}

Example 2 with AbstractStreamMetadataStore

use of io.pravega.controller.store.stream.AbstractStreamMetadataStore in project pravega by pravega.

the class ControllerGrpcAuthFocusedTest method setup.

@Before
public void setup() throws IOException {
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createInMemoryStore(EXECUTOR);
    StreamMetadataStore streamStore = StreamStoreFactory.createInMemoryStore();
    this.kvtStore = spy(KVTableStoreFactory.createInMemoryStore(streamStore, EXECUTOR));
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    SegmentHelper segmentHelper = SegmentHelperMock.getSegmentHelperMock();
    RequestTracker requestTracker = new RequestTracker(true);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    GrpcAuthHelper authHelper = new GrpcAuthHelper(true, "secret", 300);
    EventHelper helper = EventHelperMock.getEventHelperMock(EXECUTOR, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = new StreamMetadataTasks(streamStore, bucketStore, taskMetadataStore, segmentHelper, EXECUTOR, "host", authHelper, helper);
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, segmentHelper, EXECUTOR, "host", authHelper);
    kvtMetadataTasks = new TableMetadataTasks(kvtStore, segmentHelper, EXECUTOR, EXECUTOR, "host", authHelper, helper);
    StreamRequestHandler streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStore, EXECUTOR), new ScaleOperationTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStore, EXECUTOR), new DeleteStreamTask(streamMetadataTasks, streamStore, bucketStore, EXECUTOR), new TruncateStreamTask(streamMetadataTasks, streamStore, EXECUTOR), new CreateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new DeleteReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, EXECUTOR), streamStore, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, EXECUTOR), EXECUTOR);
    streamMetadataTasks.setRequestEventWriter(new ControllerEventStreamWriterMock(streamRequestHandler, EXECUTOR));
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    Cluster mockCluster = mock(Cluster.class);
    when(mockCluster.getClusterMembers()).thenReturn(Collections.singleton(new Host("localhost", 9090, null)));
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBase = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, true, 2);
    ControllerServiceGrpc.ControllerServiceImplBase controllerServiceImplBaseStrict = new ControllerServiceImpl(new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelper, EXECUTOR, mockCluster, requestTracker), authHelper, requestTracker, true, false, 2);
    PasswordAuthHandler authHandler = new PasswordAuthHandler();
    authHandler.initialize(AUTH_FILE.getAbsolutePath());
    String uniqueServerName = String.format("Test server name: %s", getClass());
    String uniqueServerNameStrict = String.format("Test server name: %sStrict", getClass());
    // Using a builder that creates a server for servicing in-process requests.
    // Also, using a direct executor which executes app code directly in transport thread. See
    // https://grpc.io/grpc-java/javadoc/io/grpc/inprocess/InProcessServerBuilder.html for more information.
    grpcServer = InProcessServerBuilder.forName(uniqueServerName).addService(ServerInterceptors.intercept(controllerServiceImplBase, new AuthInterceptor(authHandler))).directExecutor().build().start();
    grpcServerStrict = InProcessServerBuilder.forName(uniqueServerNameStrict).addService(ServerInterceptors.intercept(controllerServiceImplBaseStrict, new AuthInterceptor(authHandler))).directExecutor().build().start();
    inProcessChannel = InProcessChannelBuilder.forName(uniqueServerName).directExecutor().build();
    inProcessChannelStrict = InProcessChannelBuilder.forName(uniqueServerNameStrict).directExecutor().build();
}

Example 3 with AbstractStreamMetadataStore

use of io.pravega.controller.store.stream.AbstractStreamMetadataStore in project pravega by pravega.

the class StreamMetadataTasks method initializeStreamWriters.

public void initializeStreamWriters(final EventStreamClientFactory clientFactory, final String streamName) {
    EventHelper e = null;
    synchronized (lock) {
        if (toSetEventHelper) {
            this.eventHelper = new EventHelper(clientFactory.createEventWriter(streamName, ControllerEventProcessors.CONTROLLER_EVENT_SERIALIZER, EventWriterConfig.builder().enableConnectionPooling(true).retryAttempts(Integer.MAX_VALUE).build()), this.executor, this.eventExecutor, this.context.getHostId(), ((AbstractStreamMetadataStore) this.streamMetadataStore).getHostTaskIndex());
            toSetEventHelper = false;
            e = eventHelper;
        }
    }
    if (e != null) {
        eventHelperFuture.complete(e);
    }
}

Example 4 with AbstractStreamMetadataStore

use of io.pravega.controller.store.stream.AbstractStreamMetadataStore in project pravega by pravega.

the class StreamMetadataTasksTest method testeventHelperNPE.

@Test(timeout = 30000)
public void testeventHelperNPE() throws Exception {
    StreamMetadataStore streamMetadataStore = getStore();
    ImmutableMap<BucketStore.ServiceType, Integer> map = ImmutableMap.of(BucketStore.ServiceType.RetentionService, 1, BucketStore.ServiceType.WatermarkingService, 1);
    bucketStore = StreamStoreFactory.createInMemoryBucketStore(map);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    List<Map.Entry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 0.75));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.75, 1.0));
    EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamMetadataStore).getHostTaskIndex());
    @Cleanup StreamMetadataTasks streamMetadataTasks = new StreamMetadataTasks(streamMetadataStore, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper);
    CompletableFuture<ScaleResponse> scaleResponse = streamMetadataTasks.manualScale(SCOPE, "hellow", Collections.singletonList(1L), newRanges, 30, 0L);
    if (!scaleResponse.isDone()) {
        AbstractClientFactoryImpl clientFactory = mock(AbstractClientFactoryImpl.class);
        streamMetadataTasks.initializeStreamWriters(clientFactory, "_requestStream");
    }
    assertEquals(ScaleResponse.ScaleStreamStatus.FAILURE, scaleResponse.join().getStatus());
}

Example 5 with AbstractStreamMetadataStore

use of io.pravega.controller.store.stream.AbstractStreamMetadataStore in project pravega by pravega.

the class StreamMetadataTasksTest method sealStreamWithTxnTest.

@Test(timeout = 30000)
public void sealStreamWithTxnTest() throws Exception {
    WriterMock requestEventWriter = new WriterMock(streamMetadataTasks, executor);
    streamMetadataTasks.setRequestEventWriter(requestEventWriter);
    String streamWithTxn = "streamWithTxn";
    // region seal a stream with transactions
    long start = System.currentTimeMillis();
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    final StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(policy).build();
    streamStorePartialMock.createStream(SCOPE, streamWithTxn, config, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, streamWithTxn, State.ACTIVE, null, executor).get();
    // create txn
    VersionedTransactionData openTxn = streamTransactionMetadataTasks.createTxn(SCOPE, streamWithTxn, 10000L, 0L, 1024 * 1024L).get().getKey();
    VersionedTransactionData committingTxn = streamTransactionMetadataTasks.createTxn(SCOPE, streamWithTxn, 10000L, 0L, 1024 * 1024L).get().getKey();
    VersionedTransactionData abortingTxn = streamTransactionMetadataTasks.createTxn(SCOPE, streamWithTxn, 10000L, 0L, 1024 * 1024L).get().getKey();
    // set transaction to committing
    streamStorePartialMock.sealTransaction(SCOPE, streamWithTxn, committingTxn.getId(), true, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    // set transaction to aborting
    streamStorePartialMock.sealTransaction(SCOPE, streamWithTxn, abortingTxn.getId(), false, Optional.empty(), "", Long.MIN_VALUE, null, executor).join();
    // Mock getActiveTransactions call such that we return committing txn as OPEN txn.
    Map<UUID, ActiveTxnRecord> activeTxns = streamStorePartialMock.getActiveTxns(SCOPE, streamWithTxn, null, executor).join();
    Map<UUID, ActiveTxnRecord> retVal = activeTxns.entrySet().stream().map(tx -> {
        if (!tx.getValue().getTxnStatus().equals(TxnStatus.OPEN) && !tx.getValue().getTxnStatus().equals(TxnStatus.ABORTING)) {
            ActiveTxnRecord txRecord = tx.getValue();
            return new AbstractMap.SimpleEntry<>(tx.getKey(), new ActiveTxnRecord(txRecord.getTxCreationTimestamp(), txRecord.getLeaseExpiryTime(), txRecord.getMaxExecutionExpiryTime(), TxnStatus.OPEN));
        } else {
            return tx;
        }
    }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    doReturn(CompletableFuture.completedFuture(retVal)).when(streamStorePartialMock).getActiveTxns(eq(SCOPE), eq(streamWithTxn), any(), any());
    List<AbortEvent> abortListBefore = abortWriter.getEventList();
    streamMetadataTasks.sealStream(SCOPE, streamWithTxn, 0L);
    AssertExtensions.assertFutureThrows("seal stream did not fail processing with correct exception", processEvent(requestEventWriter), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    requestEventWriter.eventQueue.take();
    reset(streamStorePartialMock);
    // verify that the txn status is set to aborting
    VersionedTransactionData txnData = streamStorePartialMock.getTransactionData(SCOPE, streamWithTxn, openTxn.getId(), null, executor).join();
    assertEquals(txnData.getStatus(), TxnStatus.ABORTING);
    assertEquals(requestEventWriter.getEventQueue().size(), 1);
    // verify that events are posted for the abort txn.
    List<AbortEvent> abortListAfter = abortWriter.getEventList();
    assertEquals(abortListAfter.size(), abortListBefore.size() + 2);
    assertTrue(abortListAfter.stream().anyMatch(x -> x.getTxid().equals(openTxn.getId())));
    assertTrue(abortListAfter.stream().anyMatch(x -> x.getTxid().equals(abortingTxn.getId())));
    txnData = streamStorePartialMock.getTransactionData(SCOPE, streamWithTxn, committingTxn.getId(), null, executor).join();
    assertEquals(txnData.getStatus(), TxnStatus.COMMITTING);
    // Mock getActiveTransactions call such that we return some non existent transaction id so that DataNotFound is simulated.
    // returning a random transaction with list of active txns such that when its abort is attempted, Data Not Found Exception gets thrown
    retVal = new HashMap<>();
    retVal.put(UUID.randomUUID(), new ActiveTxnRecord(1L, 1L, 1L, TxnStatus.OPEN));
    doReturn(CompletableFuture.completedFuture(retVal)).when(streamStorePartialMock).getActiveTxns(eq(SCOPE), eq(streamWithTxn), any(), any());
    AssertExtensions.assertFutureThrows("seal stream did not fail processing with correct exception", processEvent(requestEventWriter), e -> Exceptions.unwrap(e) instanceof StoreException.OperationNotAllowedException);
    reset(streamStorePartialMock);
    // Now complete all existing transactions and verify that seal completes
    streamStorePartialMock.abortTransaction(SCOPE, streamWithTxn, openTxn.getId(), null, executor).join();
    streamStorePartialMock.abortTransaction(SCOPE, streamWithTxn, abortingTxn.getId(), null, executor).join();
    ((AbstractStreamMetadataStore) streamStorePartialMock).commitTransaction(SCOPE, streamWithTxn, committingTxn.getId(), null, executor).join();
    activeTxns = streamStorePartialMock.getActiveTxns(SCOPE, streamWithTxn, null, executor).join();
    assertTrue(activeTxns.isEmpty());
    assertTrue(Futures.await(processEvent(requestEventWriter)));
// endregion
}