Examples with GrpcAuthHelper io.pravega.controller.server.security.auth.GrpcAuthHelper used on opensource projects

Example 21 with GrpcAuthHelper

use of io.pravega.controller.server.security.auth.GrpcAuthHelper in project pravega by pravega.

the class StreamMetadataTasksTest method testeventHelperNPE.

@Test(timeout = 30000)
public void testeventHelperNPE() throws Exception {
    StreamMetadataStore streamMetadataStore = getStore();
    ImmutableMap<BucketStore.ServiceType, Integer> map = ImmutableMap.of(BucketStore.ServiceType.RetentionService, 1, BucketStore.ServiceType.WatermarkingService, 1);
    bucketStore = StreamStoreFactory.createInMemoryBucketStore(map);
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    List<Map.Entry<Double, Double>> newRanges = new ArrayList<>();
    newRanges.add(new AbstractMap.SimpleEntry<>(0.5, 0.75));
    newRanges.add(new AbstractMap.SimpleEntry<>(0.75, 1.0));
    EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamMetadataStore).getHostTaskIndex());
    @Cleanup StreamMetadataTasks streamMetadataTasks = new StreamMetadataTasks(streamMetadataStore, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper);
    CompletableFuture<ScaleResponse> scaleResponse = streamMetadataTasks.manualScale(SCOPE, "hellow", Collections.singletonList(1L), newRanges, 30, 0L);
    if (!scaleResponse.isDone()) {
        AbstractClientFactoryImpl clientFactory = mock(AbstractClientFactoryImpl.class);
        streamMetadataTasks.initializeStreamWriters(clientFactory, "_requestStream");
    }
    assertEquals(ScaleResponse.ScaleStreamStatus.FAILURE, scaleResponse.join().getStatus());
}

Example 22 with GrpcAuthHelper

use of io.pravega.controller.server.security.auth.GrpcAuthHelper in project pravega by pravega.

the class StreamMetadataTasksTest method concurrentCreateStreamTest.

@Test(timeout = 30000)
public void concurrentCreateStreamTest() throws Exception {
    TaskMetadataStore taskMetadataStore = spy(TaskStoreFactory.createZKStore(zkClient, executor));
    @Cleanup StreamMetadataTasks metadataTask = new StreamMetadataTasks(streamStorePartialMock, bucketStore, taskMetadataStore, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(authEnabled, "key", 300));
    final ScalingPolicy policy = ScalingPolicy.fixed(2);
    String stream = "concurrent";
    final StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(policy).build();
    CompletableFuture<Void> createStreamCalled = new CompletableFuture<>();
    CompletableFuture<Void> waitOnCreateStream = new CompletableFuture<>();
    doAnswer(x -> {
        createStreamCalled.complete(null);
        waitOnCreateStream.join();
        return x.callRealMethod();
    }).when(streamStorePartialMock).createStream(anyString(), anyString(), any(), anyLong(), any(), any());
    CompletableFuture<Controller.CreateStreamStatus.Status> createStreamFuture1 = metadataTask.createStreamRetryOnLockFailure(SCOPE, stream, config, System.currentTimeMillis(), 10, 0L);
    // wait until create stream is called. let create stream be blocked on `wait` future.
    createStreamCalled.join();
    // start a new create stream with 1 retries. this should throw lock failed exception
    // second request should fail with LockFailedException as we have not asked for a retry.
    AssertExtensions.assertFutureThrows("Lock Failed Exception should be thrown", metadataTask.createStreamRetryOnLockFailure(SCOPE, stream, config, System.currentTimeMillis(), 1, 0L), e -> Exceptions.unwrap(e) instanceof LockFailedException);
    CompletableFuture<Void> signalLockFailed = new CompletableFuture<>();
    CompletableFuture<Void> waitOnLockFailed = new CompletableFuture<>();
    // first time lock failed exception is thrown, we will complete `signalLockFailed` to indicate lock failed exception is
    // being thrown.
    // For all subsequent times we will wait on waitOnLockFailed future.
    doAnswer(x -> {
        @SuppressWarnings("unchecked") CompletableFuture<Void> future = (CompletableFuture<Void>) x.callRealMethod();
        return future.exceptionally(e -> {
            if (Exceptions.unwrap(e) instanceof LockFailedException) {
                if (!signalLockFailed.isDone()) {
                    signalLockFailed.complete(null);
                } else {
                    waitOnLockFailed.join();
                }
            }
            throw new CompletionException(e);
        });
    }).when(taskMetadataStore).lock(any(), any(), anyString(), anyString(), any(), any());
    // start a new create stream with retries.
    CompletableFuture<Controller.CreateStreamStatus.Status> createStreamFuture2 = metadataTask.createStreamRetryOnLockFailure(SCOPE, stream, config, System.currentTimeMillis(), 10, 0L);
    // wait until lock failed exception is thrown
    signalLockFailed.join();
    // now complete first createStream request
    waitOnCreateStream.complete(null);
    assertEquals(createStreamFuture1.join(), Controller.CreateStreamStatus.Status.SUCCESS);
    // now let the lock failed exception be thrown for second request for subsequent retries
    waitOnLockFailed.complete(null);
    // second request should also succeed now but with stream exists
    assertEquals(createStreamFuture2.join(), Controller.CreateStreamStatus.Status.STREAM_EXISTS);
}

Example 23 with GrpcAuthHelper

use of io.pravega.controller.server.security.auth.GrpcAuthHelper in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    StreamMetadataStore streamStore = getStore();
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    ImmutableMap<BucketStore.ServiceType, Integer> map = ImmutableMap.of(BucketStore.ServiceType.RetentionService, 1, BucketStore.ServiceType.WatermarkingService, 1);
    bucketStore = StreamStoreFactory.createInMemoryBucketStore(map);
    kvtStore = spy(getKvtStore());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper));
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300));
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStorePartialMock, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStorePartialMock, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
    commitWriter = new EventStreamWriterMock<>();
    abortWriter = new EventStreamWriterMock<>();
    streamTransactionMetadataTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE, null, executor).join();
    // stream1
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Long> sealedSegments = Collections.singletonList(1L);
    VersionedMetadata<EpochTransitionRecord> response = streamStorePartialMock.submitScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, null, null, executor).get();
    VersionedMetadata<State> state = streamStorePartialMock.getVersionedState(SCOPE, stream1, null, executor).join();
    state = streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.SCALING, state, null, executor).join();
    streamStorePartialMock.startScale(SCOPE, stream1, false, response, state, null, executor).join();
    streamStorePartialMock.scaleCreateNewEpochs(SCOPE, stream1, response, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).get();
    streamStorePartialMock.completeScale(SCOPE, stream1, response, null, executor).join();
    streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.ACTIVE, state, null, executor).get();
    // stream2
    streamStorePartialMock.createStream(SCOPE, stream2, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    streamStorePartialMock.createStream(SCOPE, stream3, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
}

Example 24 with GrpcAuthHelper

use of io.pravega.controller.server.security.auth.GrpcAuthHelper in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method writerRoutingKeyTest.

@Test(timeout = 10000)
public void writerRoutingKeyTest() throws InterruptedException {
    StreamMetadataStore streamStoreMock = StreamStoreFactory.createZKStore(zkClient, executor);
    txnTasks = new StreamTransactionMetadataTasks(streamStoreMock, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(this.authEnabled, "secret", 300));
    streamStore.createScope(SCOPE, null, executor).join();
    streamStore.createStream(SCOPE, STREAM, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), 1L, null, executor).join();
    streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    TestEventStreamWriter<CommitEvent> commitWriter = new TestEventStreamWriter<>();
    TestEventStreamWriter<AbortEvent> abortWriter = new TestEventStreamWriter<>();
    txnTasks.initializeStreamWriters(commitWriter, abortWriter);
    UUID txnId = UUID.randomUUID();
    txnTasks.writeAbortEvent(SCOPE, STREAM, 0, txnId, TxnStatus.ABORTING, 0L).join();
    Pair<String, AbortEvent> request = abortWriter.requestsReceived.take();
    assertEquals(request.getKey(), request.getValue().getKey());
    txnTasks.writeAbortEvent(new AbortEvent(SCOPE, STREAM, 0, txnId, 10L)).join();
    Pair<String, AbortEvent> request2 = abortWriter.requestsReceived.take();
    assertEquals(request2.getKey(), request2.getValue().getKey());
    // verify that both use the same key
    assertEquals(request.getKey(), request2.getKey());
    txnTasks.writeCommitEvent(SCOPE, STREAM, 0, txnId, TxnStatus.COMMITTING, 0L).join();
    Pair<String, CommitEvent> request3 = commitWriter.requestsReceived.take();
    assertEquals(request3.getKey(), request3.getValue().getKey());
    txnTasks.writeCommitEvent(new CommitEvent(SCOPE, STREAM, 0)).join();
    Pair<String, CommitEvent> request4 = commitWriter.requestsReceived.take();
    assertEquals(request4.getKey(), request4.getValue().getKey());
    // verify that both use the same key
    assertEquals(request3.getKey(), request4.getKey());
}

Example 25 with GrpcAuthHelper

use of io.pravega.controller.server.security.auth.GrpcAuthHelper in project pravega by pravega.

the class StreamTransactionMetadataTasksTest method txnPingTest.

@Test(timeout = 10000)
public void txnPingTest() throws Exception {
    // Create mock writer objects.
    EventStreamWriterMock<CommitEvent> commitWriter = new EventStreamWriterMock<>();
    EventStreamWriterMock<AbortEvent> abortWriter = new EventStreamWriterMock<>();
    StreamMetadataStore streamStoreMock = spy(StreamStoreFactory.createZKStore(zkClient, executor));
    // Create transaction tasks.
    txnTasks = new StreamTransactionMetadataTasks(streamStoreMock, SegmentHelperMock.getSegmentHelperMock(), executor, "host", new GrpcAuthHelper(this.authEnabled, "secret", 300));
    txnTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    // Create stream and scope
    streamStoreMock.createScope(SCOPE, null, executor).join();
    streamStoreMock.createStream(SCOPE, STREAM, configuration1, System.currentTimeMillis(), null, executor).join();
    streamStoreMock.setState(SCOPE, STREAM, State.ACTIVE, null, executor).join();
    // Verify Ping transaction on committing transaction.
    Pair<VersionedTransactionData, List<StreamSegmentRecord>> txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 0L).join();
    UUID txnId = txn.getKey().getId();
    txnTasks.commitTxn(SCOPE, STREAM, txnId, 0L).join();
    assertEquals(PingTxnStatus.Status.COMMITTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // complete commit of transaction.
    streamStoreMock.startCommitTransactions(SCOPE, STREAM, 100, null, executor).join();
    val record = streamStoreMock.getVersionedCommittingTransactionsRecord(SCOPE, STREAM, null, executor).join();
    streamStoreMock.completeCommitTransactions(SCOPE, STREAM, record, null, executor, Collections.emptyMap()).join();
    // verify that transaction is removed from active txn
    AssertExtensions.assertFutureThrows("Fetching Active Txn record should throw DNF", streamStoreMock.getTransactionData(SCOPE, STREAM, txnId, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
    assertEquals(PingTxnStatus.Status.COMMITTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // Verify Ping transaction on an aborting transaction.
    txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 1024 * 1024L).join();
    txnId = txn.getKey().getId();
    txnTasks.abortTxn(SCOPE, STREAM, txnId, null, 0L).join();
    assertEquals(PingTxnStatus.Status.ABORTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // now complete abort so that the transaction is removed from active txn and added to completed txn.
    streamStoreMock.abortTransaction(SCOPE, STREAM, txnId, null, executor).join();
    AssertExtensions.assertFutureThrows("Fetching Active Txn record should throw DNF", streamStoreMock.getTransactionData(SCOPE, STREAM, txnId, null, executor), e -> Exceptions.unwrap(e) instanceof StoreException.DataNotFoundException);
    assertEquals(PingTxnStatus.Status.ABORTED, txnTasks.pingTxn(SCOPE, STREAM, txnId, 10000L, 0L).join().getStatus());
    // try with a non existent transaction id
    assertEquals(PingTxnStatus.Status.UNKNOWN, txnTasks.pingTxn(SCOPE, STREAM, UUID.randomUUID(), 10000L, 0L).join().getStatus());
    // Verify max execution time.
    txnTasks.setMaxExecutionTime(1L);
    txn = txnTasks.createTxn(SCOPE, STREAM, 10000L, 0L, 1024 * 1024L).join();
    UUID tid = txn.getKey().getId();
    AssertExtensions.assertEventuallyEquals(PingTxnStatus.Status.MAX_EXECUTION_TIME_EXCEEDED, () -> txnTasks.pingTxn(SCOPE, STREAM, tid, 10000L, 0L).join().getStatus(), 10000L);
    txnTasks.setMaxExecutionTime(Duration.ofDays(Config.MAX_TXN_EXECUTION_TIMEBOUND_DAYS).toMillis());
}