Examples with RequestTracker io.pravega.common.tracing.RequestTracker used on opensource projects

Example 6 with RequestTracker

use of io.pravega.common.tracing.RequestTracker in project pravega by pravega.

the class StreamMetadataTasksTest method setup.

@Before
public void setup() throws Exception {
    zkServer = new TestingServerStarter().start();
    zkServer.start();
    zkClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000));
    zkClient.start();
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    StreamMetadataStore streamStore = getStore();
    // create a partial mock.
    streamStorePartialMock = spy(streamStore);
    ImmutableMap<BucketStore.ServiceType, Integer> map = ImmutableMap.of(BucketStore.ServiceType.RetentionService, 1, BucketStore.ServiceType.WatermarkingService, 1);
    bucketStore = StreamStoreFactory.createInMemoryBucketStore(map);
    kvtStore = spy(getKvtStore());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createZKStore(zkClient, executor);
    SegmentHelper segmentHelperMock = SegmentHelperMock.getSegmentHelperMock();
    connectionFactory = new SocketConnectionFactoryImpl(ClientConfig.builder().build());
    EventHelper helper = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    streamMetadataTasks = spy(new StreamMetadataTasks(streamStorePartialMock, bucketStore, taskMetadataStore, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300), helper));
    EventHelper helperMock = EventHelperMock.getEventHelperMock(executor, "host", ((AbstractStreamMetadataStore) streamStore).getHostTaskIndex());
    kvtMetadataTasks = spy(new TableMetadataTasks(kvtStore, segmentHelperMock, executor, executor, "host", GrpcAuthHelper.getDisabledAuthHelper(), helperMock));
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStorePartialMock, segmentHelperMock, executor, "host", new GrpcAuthHelper(authEnabled, "key", 300));
    this.streamRequestHandler = new StreamRequestHandler(new AutoScaleTask(streamMetadataTasks, streamStorePartialMock, executor), new ScaleOperationTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new SealStreamTask(streamMetadataTasks, streamTransactionMetadataTasks, streamStorePartialMock, executor), new DeleteStreamTask(streamMetadataTasks, streamStorePartialMock, bucketStore, executor), new TruncateStreamTask(streamMetadataTasks, streamStorePartialMock, executor), new CreateReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new DeleteReaderGroupTask(streamMetadataTasks, streamStorePartialMock, executor), new UpdateReaderGroupTask(streamMetadataTasks, streamStore, executor), streamStorePartialMock, new DeleteScopeTask(streamMetadataTasks, streamStore, kvtStore, kvtMetadataTasks, executor), executor);
    consumer = new ControllerService(kvtStore, kvtMetadataTasks, streamStorePartialMock, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, segmentHelperMock, executor, null, requestTracker);
    commitWriter = new EventStreamWriterMock<>();
    abortWriter = new EventStreamWriterMock<>();
    streamTransactionMetadataTasks.initializeStreamWriters(commitWriter, abortWriter);
    final ScalingPolicy policy1 = ScalingPolicy.fixed(2);
    final StreamConfiguration configuration1 = StreamConfiguration.builder().scalingPolicy(policy1).build();
    streamStorePartialMock.createScope(SCOPE, null, executor).join();
    // stream1
    long start = System.currentTimeMillis();
    streamStorePartialMock.createStream(SCOPE, stream1, configuration1, start, null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    AbstractMap.SimpleEntry<Double, Double> segment1 = new AbstractMap.SimpleEntry<>(0.5, 0.75);
    AbstractMap.SimpleEntry<Double, Double> segment2 = new AbstractMap.SimpleEntry<>(0.75, 1.0);
    List<Long> sealedSegments = Collections.singletonList(1L);
    VersionedMetadata<EpochTransitionRecord> response = streamStorePartialMock.submitScale(SCOPE, stream1, sealedSegments, Arrays.asList(segment1, segment2), start + 20, null, null, executor).get();
    VersionedMetadata<State> state = streamStorePartialMock.getVersionedState(SCOPE, stream1, null, executor).join();
    state = streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.SCALING, state, null, executor).join();
    streamStorePartialMock.startScale(SCOPE, stream1, false, response, state, null, executor).join();
    streamStorePartialMock.scaleCreateNewEpochs(SCOPE, stream1, response, null, executor).get();
    streamStorePartialMock.scaleSegmentsSealed(SCOPE, stream1, sealedSegments.stream().collect(Collectors.toMap(x -> x, x -> 0L)), response, null, executor).get();
    streamStorePartialMock.completeScale(SCOPE, stream1, response, null, executor).join();
    streamStorePartialMock.updateVersionedState(SCOPE, stream1, State.ACTIVE, state, null, executor).get();
    // stream2
    streamStorePartialMock.createStream(SCOPE, stream2, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
    streamStorePartialMock.createStream(SCOPE, stream3, configuration1, System.currentTimeMillis(), null, executor).get();
    streamStorePartialMock.setState(SCOPE, stream1, State.ACTIVE, null, executor).get();
}

Example 7 with RequestTracker

use of io.pravega.common.tracing.RequestTracker in project pravega by pravega.

the class RPCTracingHelpersTest method testInterceptors.

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testInterceptors() {
    String requestDescriptor = "createStream-myScope-myStream";
    long requestId = 1234L;
    ClientInterceptor clientInterceptor = RPCTracingHelpers.getClientInterceptor();
    RequestTracker requestTracker = new RequestTracker(true);
    // Mocking RPC elements.
    MethodDescriptor.Marshaller<Object> mockMarshaller = Mockito.mock(MethodDescriptor.Marshaller.class);
    ClientCall.Listener<Object> listener = Mockito.mock(ClientCall.Listener.class);
    ServerCall serverCall = Mockito.mock(ServerCall.class);
    ServerCallHandler serverCallHandler = Mockito.mock(ServerCallHandler.class);
    @Cleanup("shutdown") ManagedChannel channel = NettyChannelBuilder.forTarget("localhost").build();
    MethodDescriptor method = MethodDescriptor.newBuilder().setFullMethodName("createStream").setType(MethodDescriptor.MethodType.UNARY).setRequestMarshaller(mockMarshaller).setResponseMarshaller(mockMarshaller).build();
    Mockito.when(serverCall.getMethodDescriptor()).thenReturn(method);
    // Actual elements to work with.
    CallOptions callOptions = CallOptions.DEFAULT;
    Metadata headers = new Metadata();
    // Test that headers do not contain tracing-related key/values, as call options are not set.
    clientInterceptor.interceptCall(method, callOptions, channel).start(listener, headers);
    assertFalse(headers.containsKey(RPCTracingHelpers.DESCRIPTOR_HEADER));
    assertFalse(headers.containsKey(RPCTracingHelpers.ID_HEADER));
    // Check that the server interceptor handles clients not sending tracing headers and that the cache remains clean.
    ServerInterceptor serverInterceptor = RPCTracingHelpers.getServerInterceptor(requestTracker);
    serverInterceptor.interceptCall(serverCall, headers, serverCallHandler);
    assertEquals(0, requestTracker.getNumDescriptors());
    // Add call options and check that headers are correctly set.
    callOptions = callOptions.withOption(RPCTracingHelpers.REQUEST_DESCRIPTOR_CALL_OPTION, requestDescriptor).withOption(RPCTracingHelpers.REQUEST_ID_CALL_OPTION, String.valueOf(requestId));
    clientInterceptor.interceptCall(method, callOptions, channel).start(listener, headers);
    assertEquals(requestDescriptor, headers.get(RPCTracingHelpers.DESCRIPTOR_HEADER));
    assertEquals(requestId, Long.parseLong(headers.get(RPCTracingHelpers.ID_HEADER)));
    // Test that the server interceptor correctly sets these headers in the cache for further tracking.
    serverInterceptor.interceptCall(serverCall, headers, serverCallHandler);
    assertEquals(1, requestTracker.getNumDescriptors());
    assertEquals(requestId, requestTracker.getRequestIdFor(requestDescriptor));
}

Example 8 with RequestTracker

use of io.pravega.common.tracing.RequestTracker in project pravega by pravega.

the class TimeoutServiceTest method testPingOwnershipTransfer.

@Test(timeout = 30000)
public void testPingOwnershipTransfer() throws Exception {
    StreamMetadataStore streamStore2 = getStore();
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createStore(PRAVEGA_ZK_CURATOR_RESOURCE.storeClient, executor);
    SegmentHelper helperMock = SegmentHelperMock.getSegmentHelperMock();
    @Cleanup StreamMetadataTasks streamMetadataTasks2 = new StreamMetadataTasks(streamStore2, bucketStore, taskMetadataStore, helperMock, executor, "2", GrpcAuthHelper.getDisabledAuthHelper());
    @Cleanup StreamTransactionMetadataTasks streamTransactionMetadataTasks2 = new StreamTransactionMetadataTasks(streamStore2, helperMock, executor, "2", TimeoutServiceConfig.defaultConfig(), new LinkedBlockingQueue<>(5), GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks2.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    // Create TimeoutService
    TimerWheelTimeoutService timeoutService2 = (TimerWheelTimeoutService) streamTransactionMetadataTasks2.getTimeoutService();
    ControllerService controllerService2 = new ControllerService(kvtStore, kvtMetadataTasks, streamStore2, bucketStore, streamMetadataTasks2, streamTransactionMetadataTasks2, helperMock, executor, null, requestTracker);
    UUID txnId = controllerService.createTransaction(SCOPE, STREAM, LEASE, 9L).thenApply(x -> x.getKey()).join();
    VersionedTransactionData txnData = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    Assert.assertEquals(txnData.getVersion(), getVersion(0));
    Optional<Throwable> result = timeoutService.getTaskCompletionQueue().poll((long) (0.75 * LEASE), TimeUnit.MILLISECONDS);
    Assert.assertNull(result);
    TxnState txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.OPEN, txnState.getState());
    // increasing lease -> total effective lease = 3 * LEASE
    PingTxnStatus pingStatus = controllerService2.pingTransaction(SCOPE, STREAM, txnId, 2 * LEASE, 9L).join();
    Assert.assertEquals(PingTxnStatus.Status.OK, pingStatus.getStatus());
    txnData = streamStore.getTransactionData(SCOPE, STREAM, txnId, null, executor).join();
    Assert.assertEquals(txnData.getVersion(), getVersion(1));
    // timeoutService1 should believe that LEASE has expired and should get non empty completion tasks
    result = timeoutService.getTaskCompletionQueue().poll((long) (1.3 * LEASE + RETRY_DELAY), TimeUnit.MILLISECONDS);
    Assert.assertNotNull(result);
    // the txn may have been attempted to be aborted by timeoutService1 but would have failed. So txn to remain open
    txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.OPEN, txnState.getState());
    // timeoutService2 should continue to wait on lease expiry and should get empty completion tasks
    result = timeoutService2.getTaskCompletionQueue().poll(0L, TimeUnit.MILLISECONDS);
    Assert.assertNull(result);
    result = timeoutService2.getTaskCompletionQueue().poll(2 * LEASE + RETRY_DELAY, TimeUnit.MILLISECONDS);
    Assert.assertNotNull(result);
    // now txn should have moved to aborting because timeoutservice2 has initiated abort
    txnState = controllerService.checkTransactionStatus(SCOPE, STREAM, txnId, 9L).join();
    Assert.assertEquals(TxnState.State.ABORTING, txnState.getState());
}

Example 9 with RequestTracker

use of io.pravega.common.tracing.RequestTracker in project pravega by pravega.

the class TimeoutServiceTest method setUp.

@Before
public void setUp() throws Exception {
    final String hostId = "host";
    // Initialize the executor service.
    executor = ExecutorServiceHelpers.newScheduledThreadPool(5, "test");
    segmentHelper = getSegmentHelper();
    streamStore = getStore();
    HostControllerStore hostStore = HostStoreFactory.createInMemoryStore(HostMonitorConfigImpl.dummyConfig());
    TaskMetadataStore taskMetadataStore = TaskStoreFactory.createStore(PRAVEGA_ZK_CURATOR_RESOURCE.storeClient, executor);
    StreamMetrics.initialize();
    TransactionMetrics.initialize();
    streamMetadataTasks = new StreamMetadataTasks(streamStore, StreamStoreFactory.createInMemoryBucketStore(), taskMetadataStore, SegmentHelperMock.getSegmentHelperMock(), executor, hostId, GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, SegmentHelperMock.getSegmentHelperMock(), executor, hostId, TimeoutServiceConfig.defaultConfig(), new LinkedBlockingQueue<>(5), GrpcAuthHelper.getDisabledAuthHelper());
    streamTransactionMetadataTasks.initializeStreamWriters(new EventStreamWriterMock<>(), new EventStreamWriterMock<>());
    // Create TimeoutService
    timeoutService = (TimerWheelTimeoutService) streamTransactionMetadataTasks.getTimeoutService();
    BucketStore bucketStore = StreamStoreFactory.createInMemoryBucketStore();
    controllerService = new ControllerService(kvtStore, kvtMetadataTasks, streamStore, bucketStore, streamMetadataTasks, streamTransactionMetadataTasks, SegmentHelperMock.getSegmentHelperMock(), executor, null, requestTracker);
    // Create scope and stream
    streamStore.createScope(SCOPE, null, executor).join();
    StreamConfiguration streamConfiguration = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build();
    streamStore.createStream(SCOPE, STREAM, streamConfiguration, System.currentTimeMillis(), null, executor).thenCompose(x -> streamStore.setState(SCOPE, STREAM, State.ACTIVE, null, executor)).join();
}

Example 10 with RequestTracker

use of io.pravega.common.tracing.RequestTracker in project pravega by pravega.

the class WatermarkWorkflowTest method testRevisionedClientThrowsNoSuchSegmentException.

@Test(timeout = 30000L)
public void testRevisionedClientThrowsNoSuchSegmentException() {
    String scope = "scope";
    String streamName = "stream";
    StreamImpl stream = new StreamImpl(scope, streamName);
    String markStreamName = NameUtils.getMarkStreamForStream(streamName);
    SynchronizerClientFactory clientFactory = spy(SynchronizerClientFactory.class);
    ConcurrentHashMap<String, MockRevisionedStreamClient> revisionedStreamClientMap = new ConcurrentHashMap<>();
    doAnswer(x -> {
        String name = x.getArgument(0);
        return revisionedStreamClientMap.compute(name, (s, rsc) -> new MockRevisionedStreamClient(() -> streamMetadataStore.getActiveSegments(scope, name, null, executor).join().get(0).segmentId()));
    }).when(clientFactory).createRevisionedStreamClient(anyString(), any(), any());
    @Cleanup PeriodicWatermarking periodicWatermarking = new PeriodicWatermarking(streamMetadataStore, bucketStore, sp -> clientFactory, executor, new RequestTracker(false));
    streamMetadataStore.createScope(scope, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    String writer1 = "writer1";
    Map<Long, Long> map1 = ImmutableMap.of(0L, 100L, 1L, 100L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 100L, map1, null, executor).join();
    // 2. run watermarking workflow.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    MockRevisionedStreamClient revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.watermarks.size(), 1);
    Watermark watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 100L);
    // delete and recreate stream and its mark stream
    streamMetadataStore.deleteStream(scope, markStreamName, null, executor).join();
    streamMetadataStore.deleteStream(scope, streamName, null, executor).join();
    streamMetadataStore.createStream(scope, streamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(2)).timestampAggregationTimeout(10000L).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, streamName, State.ACTIVE, null, executor).join();
    streamMetadataStore.createStream(scope, markStreamName, StreamConfiguration.builder().scalingPolicy(ScalingPolicy.fixed(1)).build(), System.currentTimeMillis(), null, executor).join();
    streamMetadataStore.setState(scope, markStreamName, State.ACTIVE, null, executor).join();
    // 1. note writer1 marks
    // writer 1 reports segments 0, 1.
    map1 = ImmutableMap.of(2L, 10L, 3L, 10L);
    streamMetadataStore.noteWriterMark(scope, streamName, writer1, 10L, map1, null, executor).join();
    // 2. run watermarking workflow. this should fail and revisioned stream client should be invalidated in the cache.
    periodicWatermarking.watermark(stream).join();
    assertFalse(periodicWatermarking.checkExistsInCache(stream));
    // 3. run watermarking workflow again.
    periodicWatermarking.watermark(stream).join();
    assertTrue(periodicWatermarking.checkExistsInCache(stream));
    // verify that a watermark has been emitted.
    revisionedClient = revisionedStreamClientMap.get(markStreamName);
    assertEquals(revisionedClient.segment, 1L);
    assertEquals(revisionedClient.watermarks.size(), 1);
    watermark = revisionedClient.watermarks.get(0).getValue();
    assertEquals(watermark.getLowerTimeBound(), 10L);
}