use of org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext in project beam by apache.
the class BeamFnMapTaskExecutorTest method testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent.
@Test(timeout = ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS * 60)
public void testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent() throws Exception {
final CountDownLatch progressSentLatch = new CountDownLatch(1);
final CountDownLatch processBundleLatch = new CountDownLatch(1);
final int expectedCounterValue = 5;
final MonitoringInfo expectedMonitoringInfo = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "ExpectedCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyString").setType(MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "ExpectedPTransform").setPayload(encodeInt64Counter(expectedCounterValue)).build();
InstructionRequestHandler instructionRequestHandler = new InstructionRequestHandler() {
@Override
public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
switch(request.getRequestCase()) {
case REGISTER:
return CompletableFuture.completedFuture(responseFor(request).build());
case PROCESS_BUNDLE:
return MoreFutures.supplyAsync(() -> {
processBundleLatch.await();
return responseFor(request).setProcessBundle(BeamFnApi.ProcessBundleResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build();
});
case PROCESS_BUNDLE_PROGRESS:
progressSentLatch.countDown();
return CompletableFuture.completedFuture(responseFor(request).setProcessBundleProgress(BeamFnApi.ProcessBundleProgressResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build());
default:
throw new RuntimeException("Reached unexpected code path");
}
}
@Override
public void registerProcessBundleDescriptor(ProcessBundleDescriptor descriptor) {
}
@Override
public void close() {
}
};
Map<String, DataflowStepContext> stepContextMap = new HashMap<>();
stepContextMap.put("ExpectedPTransform", generateDataflowStepContext("Expected"));
RegisterAndProcessBundleOperation processOperation = new RegisterAndProcessBundleOperation(IdGenerators.decrementingLongs(), instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), stepContextMap, ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
BeamFnMapTaskExecutor mapTaskExecutor = BeamFnMapTaskExecutor.forOperations(ImmutableList.of(readOperation, grpcPortWriteOperation, processOperation), executionStateTracker);
// Launch the BeamFnMapTaskExecutor and wait until we are sure there has been one
// tentative update
CompletionStage<Void> doneFuture = MoreFutures.runAsync(mapTaskExecutor::execute);
progressSentLatch.await();
Iterable<CounterUpdate> metricsCounterUpdates = Collections.emptyList();
while (Iterables.size(metricsCounterUpdates) == 0) {
Thread.sleep(ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS);
metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
}
// Get the final metrics
processBundleLatch.countDown();
MoreFutures.get(doneFuture);
metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
assertThat(Iterables.size(metricsCounterUpdates), equalTo(1));
CounterUpdate resultCounter = metricsCounterUpdates.iterator().next();
assertTrue(new CounterHamcrestMatchers.CounterUpdateIntegerValueMatcher(expectedCounterValue).matches(resultCounter));
assertEquals("ExpectedCounter", resultCounter.getStructuredNameAndMetadata().getName().getName());
}
use of org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext in project beam by apache.
the class RegisterAndProcessBundleOperationTest method testProcessingBundleHandlesUserStateRequests.
@Test
public void testProcessingBundleHandlesUserStateRequests() throws Exception {
IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
InMemoryStateInternals<ByteString> stateInternals = InMemoryStateInternals.forKey(ByteString.EMPTY);
DataflowStepContext mockStepContext = mock(DataflowStepContext.class);
DataflowStepContext mockUserStepContext = mock(DataflowStepContext.class);
when(mockStepContext.namespacedToUser()).thenReturn(mockUserStepContext);
when(mockUserStepContext.stateInternals()).thenReturn(stateInternals);
InstructionRequestHandler instructionRequestHandler = new TestInstructionRequestHandler() {
@Override
public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
switch(request.getRequestCase()) {
case REGISTER:
return CompletableFuture.completedFuture(responseFor(request).build());
case PROCESS_BUNDLE:
return MoreFutures.supplyAsync(() -> {
StateRequest partialRequest = StateRequest.newBuilder().setStateKey(StateKey.newBuilder().setBagUserState(StateKey.BagUserState.newBuilder().setTransformId("testPTransformId").setWindow(ByteString.EMPTY).setUserStateId("testUserStateId"))).buildPartial();
StateRequest get = partialRequest.toBuilder().setGet(StateGetRequest.getDefaultInstance()).build();
StateRequest clear = partialRequest.toBuilder().setClear(StateClearRequest.getDefaultInstance()).build();
StateRequest append = partialRequest.toBuilder().setAppend(StateAppendRequest.newBuilder().setData(ByteString.copyFromUtf8("ABC"))).build();
StateRequestHandler stateHandler = stateHandlerCaptor.getValue();
StateResponse.Builder getWhenEmptyResponse = MoreFutures.get(stateHandler.handle(get));
assertEquals(ByteString.EMPTY, getWhenEmptyResponse.getGet().getData());
StateResponse.Builder appendWhenEmptyResponse = MoreFutures.get(stateHandler.handle(append));
assertNotNull(appendWhenEmptyResponse);
StateResponse.Builder appendWhenEmptyResponse2 = MoreFutures.get(stateHandler.handle(append));
assertNotNull(appendWhenEmptyResponse2);
StateResponse.Builder getWhenHasValueResponse = MoreFutures.get(stateHandler.handle(get));
assertEquals(ByteString.copyFromUtf8("ABC").concat(ByteString.copyFromUtf8("ABC")), getWhenHasValueResponse.getGet().getData());
StateResponse.Builder clearResponse = MoreFutures.get(stateHandler.handle(clear));
assertNotNull(clearResponse);
return responseFor(request).build();
});
default:
// block forever
return new CompletableFuture<>();
}
}
};
RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of("testPTransformId", mockStepContext), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
operation.start();
verify(mockBeamFnStateDelegator).registerForProcessBundleInstructionId(eq("778"), stateHandlerCaptor.capture());
// This method blocks till the requests are completed
operation.finish();
// Ensure that the number of reigstrations matches the number of deregistrations
assertEquals(stateServiceRegisterCounter.get(), stateServiceDeregisterCounter.get());
assertEquals(0, stateServiceAbortCounter.get());
}
use of org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext in project beam by apache.
the class RegisterAndProcessBundleOperationTest method testProcessingBundleHandlesMultimapSideInputRequests.
@Test
public void testProcessingBundleHandlesMultimapSideInputRequests() throws Exception {
IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
DataflowStepContext mockStepContext = mock(DataflowStepContext.class);
DataflowStepContext mockUserStepContext = mock(DataflowStepContext.class);
when(mockStepContext.namespacedToUser()).thenReturn(mockUserStepContext);
CountDownLatch waitForStateHandler = new CountDownLatch(1);
// Issues state calls to the Runner after a process bundle request is sent.
InstructionRequestHandler fakeClient = new TestInstructionRequestHandler() {
@Override
public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
switch(request.getRequestCase()) {
case REGISTER:
return CompletableFuture.completedFuture(responseFor(request).build());
case PROCESS_BUNDLE:
return MoreFutures.supplyAsync(() -> {
StateKey getKey = StateKey.newBuilder().setMultimapSideInput(StateKey.MultimapSideInput.newBuilder().setTransformId("testPTransformId").setSideInputId("testSideInputId").setWindow(ByteString.copyFrom(CoderUtils.encodeToByteArray(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE))).setKey(ByteString.copyFrom(CoderUtils.encodeToByteArray(ByteArrayCoder.of(), "ABC".getBytes(StandardCharsets.UTF_8), Coder.Context.NESTED)))).build();
StateRequest getRequest = StateRequest.newBuilder().setStateKey(getKey).setGet(StateGetRequest.getDefaultInstance()).build();
waitForStateHandler.await();
StateRequestHandler stateHandler = stateHandlerCaptor.getValue();
StateResponse.Builder getResponse = MoreFutures.get(stateHandler.handle(getRequest));
assertEquals(encodeAndConcat(Arrays.asList("X", "Y", "Z"), StringUtf8Coder.of()), getResponse.getGet().getData());
return responseFor(request).build();
});
default:
// block forever on other request types
return new CompletableFuture<>();
}
}
};
SideInputReader fakeSideInputReader = new SideInputReader() {
@Override
@Nullable
public <T> T get(PCollectionView<T> view, BoundedWindow window) {
assertEquals(GlobalWindow.INSTANCE, window);
assertEquals("testSideInputId", view.getTagInternal().getId());
return (T) InMemoryMultimapSideInputView.fromIterable(ByteArrayCoder.of(), ImmutableList.of(KV.of("ABC".getBytes(StandardCharsets.UTF_8), "X"), KV.of("ABC".getBytes(StandardCharsets.UTF_8), "Y"), KV.of("ABC".getBytes(StandardCharsets.UTF_8), "Z")));
}
@Override
public <T> boolean contains(PCollectionView<T> view) {
return "testSideInputId".equals(view.getTagInternal().getId());
}
@Override
public boolean isEmpty() {
return false;
}
};
RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, fakeClient, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of("testPTransformId", mockStepContext), ImmutableMap.of("testPTransformId", fakeSideInputReader), ImmutableTable.of("testPTransformId", "testSideInputId", DataflowPortabilityPCollectionView.with(new TupleTag<>("testSideInputId"), FullWindowedValueCoder.of(KvCoder.of(ByteArrayCoder.of(), StringUtf8Coder.of()), GlobalWindow.Coder.INSTANCE))), ImmutableMap.of(), mockContext);
operation.start();
verify(mockBeamFnStateDelegator).registerForProcessBundleInstructionId(eq("778"), stateHandlerCaptor.capture());
waitForStateHandler.countDown();
// This method blocks till the requests are completed
operation.finish();
// Ensure that the number of reigstrations matches the number of deregistrations
assertEquals(stateServiceRegisterCounter.get(), stateServiceDeregisterCounter.get());
assertEquals(0, stateServiceAbortCounter.get());
}
use of org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext in project beam by apache.
the class UserDistributionMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived.
@Test
public void testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived() throws Exception {
Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
NameContext nc = NameContext.create("anyStageName", "anyOriginalName", "anySystemName", "anyUserName");
DataflowStepContext dsc = mock(DataflowStepContext.class);
when(dsc.getNameContext()).thenReturn(nc);
stepContextMapping.put("anyValue", dsc);
MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.USER_DISTRIBUTION_INT64).setType(TypeUrns.DISTRIBUTION_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.NAME, "anyName").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyNamespace").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").setPayload(encodeInt64Distribution(DistributionData.create(2L, /* sum */
1L, /* count */
3L, /* min */
4L))).build();
UserDistributionMonitoringInfoToCounterUpdateTransformer testObject = new UserDistributionMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
CounterUpdate result = testObject.transform(monitoringInfo);
assertNotNull(result);
assertEqualsAsJson("{cumulative:true, distribution:{count:{highBits:0, lowBits:1}, " + "max:{highBits:0, lowBits:4}, min:{highBits:0, lowBits:3}, " + "sum:{highBits:0, lowBits:2}}, " + "structuredNameAndMetadata:{metadata:{kind:'DISTRIBUTION'}, " + "name:{name:'anyName', origin:'USER', originNamespace:'anyNamespace', " + "originalStepName:'anyOriginalName'}}}", result);
}
use of org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext in project beam by apache.
the class ExecutionTimeMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsNullIfMonitoringInfoWithUnknownPTransformLabelPresent.
@Test
public void testTransformReturnsNullIfMonitoringInfoWithUnknownPTransformLabelPresent() {
Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
ExecutionTimeMonitoringInfoToCounterUpdateTransformer testObject = new ExecutionTimeMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.PROCESS_BUNDLE_MSECS).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").build();
assertNull(testObject.transform(monitoringInfo));
}
Aggregations