Examples with InstructionRequest org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest used on opensource projects

Example 1 with InstructionRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest in project beam by apache.

the class FnHarnessTest method testLaunchFnHarnessAndTeardownCleanly.

@Test
// failure will cause test to timeout.
@SuppressWarnings("FutureReturnValueIgnored")
public void testLaunchFnHarnessAndTeardownCleanly() throws Exception {
    Function<String, String> environmentVariableMock = mock(Function.class);
    PipelineOptions options = PipelineOptionsFactory.create();
    when(environmentVariableMock.apply("HARNESS_ID")).thenReturn("id");
    when(environmentVariableMock.apply("PIPELINE_OPTIONS")).thenReturn(PipelineOptionsTranslation.toJson(options));
    List<BeamFnApi.LogEntry> logEntries = new ArrayList<>();
    List<BeamFnApi.InstructionResponse> instructionResponses = mock(List.class);
    BeamFnLoggingGrpc.BeamFnLoggingImplBase loggingService = new BeamFnLoggingGrpc.BeamFnLoggingImplBase() {

        @Override
        public StreamObserver<BeamFnApi.LogEntry.List> logging(StreamObserver<LogControl> responseObserver) {
            return TestStreams.withOnNext((BeamFnApi.LogEntry.List entries) -> logEntries.addAll(entries.getLogEntriesList())).withOnCompleted(responseObserver::onCompleted).build();
        }
    };
    BeamFnControlGrpc.BeamFnControlImplBase controlService = new BeamFnControlGrpc.BeamFnControlImplBase() {

        @Override
        public StreamObserver<InstructionResponse> control(StreamObserver<InstructionRequest> responseObserver) {
            CountDownLatch waitForResponses = new CountDownLatch(1);
            options.as(GcsOptions.class).getExecutorService().submit(() -> {
                responseObserver.onNext(INSTRUCTION_REQUEST);
                Uninterruptibles.awaitUninterruptibly(waitForResponses);
                responseObserver.onCompleted();
            });
            return TestStreams.withOnNext((InstructionResponse t) -> {
                instructionResponses.add(t);
                waitForResponses.countDown();
            }).withOnCompleted(waitForResponses::countDown).build();
        }
    };
    Server loggingServer = ServerBuilder.forPort(0).addService(loggingService).build();
    loggingServer.start();
    try {
        Server controlServer = ServerBuilder.forPort(0).addService(controlService).build();
        controlServer.start();
        try {
            Endpoints.ApiServiceDescriptor loggingDescriptor = Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:" + loggingServer.getPort()).build();
            Endpoints.ApiServiceDescriptor controlDescriptor = Endpoints.ApiServiceDescriptor.newBuilder().setUrl("localhost:" + controlServer.getPort()).build();
            when(environmentVariableMock.apply("LOGGING_API_SERVICE_DESCRIPTOR")).thenReturn(TextFormat.printToString(loggingDescriptor));
            when(environmentVariableMock.apply("CONTROL_API_SERVICE_DESCRIPTOR")).thenReturn(TextFormat.printToString(controlDescriptor));
            FnHarness.main(environmentVariableMock);
        } finally {
            controlServer.shutdownNow();
        }
    } finally {
        loggingServer.shutdownNow();
    }
    // Verify that we first run onStartup functions before even reading the environment, and that
    // we then call beforeProcessing functions before executing instructions.
    InOrder inOrder = inOrder(onStartupMock, beforeProcessingMock, environmentVariableMock, instructionResponses);
    inOrder.verify(onStartupMock).run();
    inOrder.verify(environmentVariableMock, atLeastOnce()).apply(any());
    inOrder.verify(beforeProcessingMock).accept(any());
    inOrder.verify(instructionResponses).add(INSTRUCTION_RESPONSE);
}

Example 2 with InstructionRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest in project beam by apache.

the class BeamFnMapTaskExecutorTest method testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent.

@Test(timeout = ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS * 60)
public void testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent() throws Exception {
    final CountDownLatch progressSentLatch = new CountDownLatch(1);
    final CountDownLatch processBundleLatch = new CountDownLatch(1);
    final int expectedCounterValue = 5;
    final MonitoringInfo expectedMonitoringInfo = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "ExpectedCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyString").setType(MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "ExpectedPTransform").setPayload(encodeInt64Counter(expectedCounterValue)).build();
    InstructionRequestHandler instructionRequestHandler = new InstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    return MoreFutures.supplyAsync(() -> {
                        processBundleLatch.await();
                        return responseFor(request).setProcessBundle(BeamFnApi.ProcessBundleResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build();
                    });
                case PROCESS_BUNDLE_PROGRESS:
                    progressSentLatch.countDown();
                    return CompletableFuture.completedFuture(responseFor(request).setProcessBundleProgress(BeamFnApi.ProcessBundleProgressResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build());
                default:
                    throw new RuntimeException("Reached unexpected code path");
            }
        }

        @Override
        public void registerProcessBundleDescriptor(ProcessBundleDescriptor descriptor) {
        }

        @Override
        public void close() {
        }
    };
    Map<String, DataflowStepContext> stepContextMap = new HashMap<>();
    stepContextMap.put("ExpectedPTransform", generateDataflowStepContext("Expected"));
    RegisterAndProcessBundleOperation processOperation = new RegisterAndProcessBundleOperation(IdGenerators.decrementingLongs(), instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), stepContextMap, ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    BeamFnMapTaskExecutor mapTaskExecutor = BeamFnMapTaskExecutor.forOperations(ImmutableList.of(readOperation, grpcPortWriteOperation, processOperation), executionStateTracker);
    // Launch the BeamFnMapTaskExecutor and wait until we are sure there has been one
    // tentative update
    CompletionStage<Void> doneFuture = MoreFutures.runAsync(mapTaskExecutor::execute);
    progressSentLatch.await();
    Iterable<CounterUpdate> metricsCounterUpdates = Collections.emptyList();
    while (Iterables.size(metricsCounterUpdates) == 0) {
        Thread.sleep(ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS);
        metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    }
    // Get the final metrics
    processBundleLatch.countDown();
    MoreFutures.get(doneFuture);
    metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    assertThat(Iterables.size(metricsCounterUpdates), equalTo(1));
    CounterUpdate resultCounter = metricsCounterUpdates.iterator().next();
    assertTrue(new CounterHamcrestMatchers.CounterUpdateIntegerValueMatcher(expectedCounterValue).matches(resultCounter));
    assertEquals("ExpectedCounter", resultCounter.getStructuredNameAndMetadata().getName().getName());
}

Example 3 with InstructionRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testProcessingBundleBlocksOnFinishWithError.

@Test
public void testProcessingBundleBlocksOnFinishWithError() throws Exception {
    List<BeamFnApi.InstructionRequest> requests = new ArrayList<>();
    IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
    ExecutorService executorService = Executors.newCachedThreadPool();
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, new TestInstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            requests.add(request);
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    CompletableFuture<InstructionResponse> responseFuture = new CompletableFuture<>();
                    executorService.submit(() -> {
                        // Purposefully sleep simulating SDK harness doing work
                        Thread.sleep(100);
                        responseFuture.complete(responseFor(request).setError("error").build());
                        return null;
                    });
                    return responseFuture;
                default:
                    // Anything else hangs; nothing else should be blocking
                    return new CompletableFuture<>();
            }
        }
    }, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    operation.start();
    // This method blocks till the requests are completed
    operation.finish();
    assertEquals(true, operation.hasFailed());
    // Ensure that the messages were received
    assertEquals(requests.get(0), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("777").setRegister(REGISTER_REQUEST).build());
    assertEquals(requests.get(1), BeamFnApi.InstructionRequest.newBuilder().setInstructionId("778").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("555")).build());
}

Example 4 with InstructionRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testProcessingBundleHandlesUserStateRequests.

@Test
public void testProcessingBundleHandlesUserStateRequests() throws Exception {
    IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
    InMemoryStateInternals<ByteString> stateInternals = InMemoryStateInternals.forKey(ByteString.EMPTY);
    DataflowStepContext mockStepContext = mock(DataflowStepContext.class);
    DataflowStepContext mockUserStepContext = mock(DataflowStepContext.class);
    when(mockStepContext.namespacedToUser()).thenReturn(mockUserStepContext);
    when(mockUserStepContext.stateInternals()).thenReturn(stateInternals);
    InstructionRequestHandler instructionRequestHandler = new TestInstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    return MoreFutures.supplyAsync(() -> {
                        StateRequest partialRequest = StateRequest.newBuilder().setStateKey(StateKey.newBuilder().setBagUserState(StateKey.BagUserState.newBuilder().setTransformId("testPTransformId").setWindow(ByteString.EMPTY).setUserStateId("testUserStateId"))).buildPartial();
                        StateRequest get = partialRequest.toBuilder().setGet(StateGetRequest.getDefaultInstance()).build();
                        StateRequest clear = partialRequest.toBuilder().setClear(StateClearRequest.getDefaultInstance()).build();
                        StateRequest append = partialRequest.toBuilder().setAppend(StateAppendRequest.newBuilder().setData(ByteString.copyFromUtf8("ABC"))).build();
                        StateRequestHandler stateHandler = stateHandlerCaptor.getValue();
                        StateResponse.Builder getWhenEmptyResponse = MoreFutures.get(stateHandler.handle(get));
                        assertEquals(ByteString.EMPTY, getWhenEmptyResponse.getGet().getData());
                        StateResponse.Builder appendWhenEmptyResponse = MoreFutures.get(stateHandler.handle(append));
                        assertNotNull(appendWhenEmptyResponse);
                        StateResponse.Builder appendWhenEmptyResponse2 = MoreFutures.get(stateHandler.handle(append));
                        assertNotNull(appendWhenEmptyResponse2);
                        StateResponse.Builder getWhenHasValueResponse = MoreFutures.get(stateHandler.handle(get));
                        assertEquals(ByteString.copyFromUtf8("ABC").concat(ByteString.copyFromUtf8("ABC")), getWhenHasValueResponse.getGet().getData());
                        StateResponse.Builder clearResponse = MoreFutures.get(stateHandler.handle(clear));
                        assertNotNull(clearResponse);
                        return responseFor(request).build();
                    });
                default:
                    // block forever
                    return new CompletableFuture<>();
            }
        }
    };
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of("testPTransformId", mockStepContext), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    operation.start();
    verify(mockBeamFnStateDelegator).registerForProcessBundleInstructionId(eq("778"), stateHandlerCaptor.capture());
    // This method blocks till the requests are completed
    operation.finish();
    // Ensure that the number of reigstrations matches the number of deregistrations
    assertEquals(stateServiceRegisterCounter.get(), stateServiceDeregisterCounter.get());
    assertEquals(0, stateServiceAbortCounter.get());
}

Example 5 with InstructionRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testProcessingBundleHandlesMultimapSideInputRequests.

@Test
public void testProcessingBundleHandlesMultimapSideInputRequests() throws Exception {
    IdGenerator idGenerator = makeIdGeneratorStartingFrom(777L);
    DataflowStepContext mockStepContext = mock(DataflowStepContext.class);
    DataflowStepContext mockUserStepContext = mock(DataflowStepContext.class);
    when(mockStepContext.namespacedToUser()).thenReturn(mockUserStepContext);
    CountDownLatch waitForStateHandler = new CountDownLatch(1);
    // Issues state calls to the Runner after a process bundle request is sent.
    InstructionRequestHandler fakeClient = new TestInstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    return MoreFutures.supplyAsync(() -> {
                        StateKey getKey = StateKey.newBuilder().setMultimapSideInput(StateKey.MultimapSideInput.newBuilder().setTransformId("testPTransformId").setSideInputId("testSideInputId").setWindow(ByteString.copyFrom(CoderUtils.encodeToByteArray(GlobalWindow.Coder.INSTANCE, GlobalWindow.INSTANCE))).setKey(ByteString.copyFrom(CoderUtils.encodeToByteArray(ByteArrayCoder.of(), "ABC".getBytes(StandardCharsets.UTF_8), Coder.Context.NESTED)))).build();
                        StateRequest getRequest = StateRequest.newBuilder().setStateKey(getKey).setGet(StateGetRequest.getDefaultInstance()).build();
                        waitForStateHandler.await();
                        StateRequestHandler stateHandler = stateHandlerCaptor.getValue();
                        StateResponse.Builder getResponse = MoreFutures.get(stateHandler.handle(getRequest));
                        assertEquals(encodeAndConcat(Arrays.asList("X", "Y", "Z"), StringUtf8Coder.of()), getResponse.getGet().getData());
                        return responseFor(request).build();
                    });
                default:
                    // block forever on other request types
                    return new CompletableFuture<>();
            }
        }
    };
    SideInputReader fakeSideInputReader = new SideInputReader() {

        @Override
        @Nullable
        public <T> T get(PCollectionView<T> view, BoundedWindow window) {
            assertEquals(GlobalWindow.INSTANCE, window);
            assertEquals("testSideInputId", view.getTagInternal().getId());
            return (T) InMemoryMultimapSideInputView.fromIterable(ByteArrayCoder.of(), ImmutableList.of(KV.of("ABC".getBytes(StandardCharsets.UTF_8), "X"), KV.of("ABC".getBytes(StandardCharsets.UTF_8), "Y"), KV.of("ABC".getBytes(StandardCharsets.UTF_8), "Z")));
        }

        @Override
        public <T> boolean contains(PCollectionView<T> view) {
            return "testSideInputId".equals(view.getTagInternal().getId());
        }

        @Override
        public boolean isEmpty() {
            return false;
        }
    };
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(idGenerator, fakeClient, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of("testPTransformId", mockStepContext), ImmutableMap.of("testPTransformId", fakeSideInputReader), ImmutableTable.of("testPTransformId", "testSideInputId", DataflowPortabilityPCollectionView.with(new TupleTag<>("testSideInputId"), FullWindowedValueCoder.of(KvCoder.of(ByteArrayCoder.of(), StringUtf8Coder.of()), GlobalWindow.Coder.INSTANCE))), ImmutableMap.of(), mockContext);
    operation.start();
    verify(mockBeamFnStateDelegator).registerForProcessBundleInstructionId(eq("778"), stateHandlerCaptor.capture());
    waitForStateHandler.countDown();
    // This method blocks till the requests are completed
    operation.finish();
    // Ensure that the number of reigstrations matches the number of deregistrations
    assertEquals(stateServiceRegisterCounter.get(), stateServiceDeregisterCounter.get());
    assertEquals(0, stateServiceAbortCounter.get());
}