Examples with InstructionRequestHandler org.apache.beam.runners.fnexecution.control.InstructionRequestHandler used on opensource projects

Example 1 with InstructionRequestHandler

use of org.apache.beam.runners.fnexecution.control.InstructionRequestHandler in project beam by apache.

the class EmbeddedSdkHarness method before.

@Override
protected void before() throws Exception {
    InProcessServerFactory serverFactory = InProcessServerFactory.create();
    executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setDaemon(true).build());
    ControlClientPool clientPool = MapControlClientPool.create();
    FnApiControlClientPoolService clientPoolService = FnApiControlClientPoolService.offeringClientsToPool(clientPool.getSink(), GrpcContextHeaderAccessorProvider.getHeaderAccessor());
    loggingServer = GrpcFnServer.allocatePortAndCreateFor(GrpcLoggingService.forWriter(Slf4jLogWriter.getDefault()), serverFactory);
    dataServer = GrpcFnServer.allocatePortAndCreateFor(GrpcDataService.create(PipelineOptionsFactory.create(), executor, OutboundObserverFactory.serverDirect()), serverFactory);
    controlServer = GrpcFnServer.allocatePortAndCreateFor(clientPoolService, serverFactory);
    InstructionRequestHandler requestHandler = EmbeddedEnvironmentFactory.create(PipelineOptionsFactory.create(), loggingServer, controlServer, clientPool.getSource()).createEnvironment(Environment.getDefaultInstance(), "embedded_worker").getInstructionRequestHandler();
    client = SdkHarnessClient.usingFnApiClient(requestHandler, dataServer.getService());
}

Example 2 with InstructionRequestHandler

use of org.apache.beam.runners.fnexecution.control.InstructionRequestHandler in project beam by apache.

the class DockerEnvironmentFactory method createEnvironment.

/**
 * Creates a new, active {@link RemoteEnvironment} backed by a local Docker container.
 */
@Override
public RemoteEnvironment createEnvironment(Environment environment, String workerId) throws Exception {
    Preconditions.checkState(environment.getUrn().equals(BeamUrns.getUrn(RunnerApi.StandardEnvironments.Environments.DOCKER)), "The passed environment does not contain a DockerPayload.");
    final RunnerApi.DockerPayload dockerPayload = RunnerApi.DockerPayload.parseFrom(environment.getPayload());
    // Prepare docker invocation.
    String containerImage = dockerPayload.getContainerImage();
    // TODO: https://issues.apache.org/jira/browse/BEAM-4148 The default service address will not
    // work for Docker for Mac.
    String provisionEndpoint = provisioningServiceServer.getApiServiceDescriptor().getUrl();
    ImmutableList.Builder<String> dockerOptsBuilder = ImmutableList.<String>builder().addAll(gcsCredentialArgs()).add("--network=host").add("--env=DOCKER_MAC_CONTAINER=" + System.getenv("DOCKER_MAC_CONTAINER"));
    final boolean retainDockerContainer = pipelineOptions.as(ManualDockerEnvironmentOptions.class).getRetainDockerContainers();
    String semiPersistDir = pipelineOptions.as(RemoteEnvironmentOptions.class).getSemiPersistDir();
    ImmutableList.Builder<String> argsBuilder = ImmutableList.<String>builder().add(String.format("--id=%s", workerId)).add(String.format("--provision_endpoint=%s", provisionEndpoint));
    if (semiPersistDir != null) {
        argsBuilder.add(String.format("--semi_persist_dir=%s", semiPersistDir));
    }
    LOG.debug("Creating Docker Container with ID {}", workerId);
    // Wrap the blocking call to clientSource.get in case an exception is thrown.
    String containerId = null;
    InstructionRequestHandler instructionHandler = null;
    try {
        containerId = docker.runImage(containerImage, dockerOptsBuilder.build(), argsBuilder.build());
        LOG.debug("Created Docker Container with Container ID {}", containerId);
        // Wait on a client from the gRPC server.
        while (instructionHandler == null) {
            try {
                // If the docker is not alive anymore, we abort.
                if (!docker.isContainerRunning(containerId)) {
                    IllegalStateException illegalStateException = new IllegalStateException(String.format("No container running for id %s", containerId));
                    try {
                        String containerLogs = docker.getContainerLogs(containerId);
                        LOG.error("Docker container {} logs:\n{}", containerId, containerLogs);
                    } catch (Exception getLogsException) {
                        illegalStateException.addSuppressed(getLogsException);
                    }
                    throw illegalStateException;
                }
                instructionHandler = clientSource.take(workerId, Duration.ofSeconds(5));
            } catch (TimeoutException timeoutEx) {
                LOG.info("Still waiting for startup of environment {} for worker id {}", dockerPayload.getContainerImage(), workerId);
            } catch (InterruptedException interruptEx) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(interruptEx);
            }
        }
    } catch (Exception e) {
        if (containerId != null) {
            // Kill the launched docker container if we can't retrieve a client for it.
            try {
                docker.killContainer(containerId);
                if (!retainDockerContainer) {
                    docker.removeContainer(containerId);
                }
            } catch (Exception dockerException) {
                e.addSuppressed(dockerException);
            }
        }
        throw e;
    }
    return DockerContainerEnvironment.create(docker, environment, containerId, instructionHandler, retainDockerContainer);
}

Example 3 with InstructionRequestHandler

use of org.apache.beam.runners.fnexecution.control.InstructionRequestHandler in project beam by apache.

the class ProcessEnvironmentFactory method createEnvironment.

/**
 * Creates a new, active {@link RemoteEnvironment} backed by a forked process.
 */
@Override
public RemoteEnvironment createEnvironment(Environment environment, String workerId) throws Exception {
    Preconditions.checkState(environment.getUrn().equals(BeamUrns.getUrn(RunnerApi.StandardEnvironments.Environments.PROCESS)), "The passed environment does not contain a ProcessPayload.");
    final RunnerApi.ProcessPayload processPayload = RunnerApi.ProcessPayload.parseFrom(environment.getPayload());
    String executable = processPayload.getCommand();
    String provisionEndpoint = provisioningServiceServer.getApiServiceDescriptor().getUrl();
    String semiPersistDir = pipelineOptions.as(RemoteEnvironmentOptions.class).getSemiPersistDir();
    ImmutableList.Builder<String> argsBuilder = ImmutableList.<String>builder().add(String.format("--id=%s", workerId)).add(String.format("--provision_endpoint=%s", provisionEndpoint));
    if (semiPersistDir != null) {
        argsBuilder.add(String.format("--semi_persist_dir=%s", semiPersistDir));
    }
    LOG.debug("Creating Process for worker ID {}", workerId);
    // Wrap the blocking call to clientSource.get in case an exception is thrown.
    InstructionRequestHandler instructionHandler = null;
    try {
        ProcessManager.RunningProcess process = processManager.startProcess(workerId, executable, argsBuilder.build(), processPayload.getEnvMap());
        // Wait on a client from the gRPC server.
        while (instructionHandler == null) {
            try {
                // If the process is not alive anymore, we abort.
                process.isAliveOrThrow();
                instructionHandler = clientSource.take(workerId, Duration.ofSeconds(5));
            } catch (TimeoutException timeoutEx) {
                LOG.info("Still waiting for startup of environment '{}' for worker id {}", processPayload.getCommand(), workerId);
            } catch (InterruptedException interruptEx) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(interruptEx);
            }
        }
    } catch (Exception e) {
        try {
            processManager.stopProcess(workerId);
        } catch (Exception processKillException) {
            e.addSuppressed(processKillException);
        }
        throw e;
    }
    return ProcessEnvironment.create(processManager, environment, workerId, instructionHandler);
}

Example 4 with InstructionRequestHandler

use of org.apache.beam.runners.fnexecution.control.InstructionRequestHandler in project beam by apache.

the class BeamFnMapTaskExecutorTest method testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent.

@Test(timeout = ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS * 60)
public void testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent() throws Exception {
    final CountDownLatch progressSentLatch = new CountDownLatch(1);
    final CountDownLatch processBundleLatch = new CountDownLatch(1);
    final int expectedCounterValue = 5;
    final MonitoringInfo expectedMonitoringInfo = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "ExpectedCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyString").setType(MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "ExpectedPTransform").setPayload(encodeInt64Counter(expectedCounterValue)).build();
    InstructionRequestHandler instructionRequestHandler = new InstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    return MoreFutures.supplyAsync(() -> {
                        processBundleLatch.await();
                        return responseFor(request).setProcessBundle(BeamFnApi.ProcessBundleResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build();
                    });
                case PROCESS_BUNDLE_PROGRESS:
                    progressSentLatch.countDown();
                    return CompletableFuture.completedFuture(responseFor(request).setProcessBundleProgress(BeamFnApi.ProcessBundleProgressResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build());
                default:
                    throw new RuntimeException("Reached unexpected code path");
            }
        }

        @Override
        public void registerProcessBundleDescriptor(ProcessBundleDescriptor descriptor) {
        }

        @Override
        public void close() {
        }
    };
    Map<String, DataflowStepContext> stepContextMap = new HashMap<>();
    stepContextMap.put("ExpectedPTransform", generateDataflowStepContext("Expected"));
    RegisterAndProcessBundleOperation processOperation = new RegisterAndProcessBundleOperation(IdGenerators.decrementingLongs(), instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), stepContextMap, ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    BeamFnMapTaskExecutor mapTaskExecutor = BeamFnMapTaskExecutor.forOperations(ImmutableList.of(readOperation, grpcPortWriteOperation, processOperation), executionStateTracker);
    // Launch the BeamFnMapTaskExecutor and wait until we are sure there has been one
    // tentative update
    CompletionStage<Void> doneFuture = MoreFutures.runAsync(mapTaskExecutor::execute);
    progressSentLatch.await();
    Iterable<CounterUpdate> metricsCounterUpdates = Collections.emptyList();
    while (Iterables.size(metricsCounterUpdates) == 0) {
        Thread.sleep(ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS);
        metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    }
    // Get the final metrics
    processBundleLatch.countDown();
    MoreFutures.get(doneFuture);
    metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    assertThat(Iterables.size(metricsCounterUpdates), equalTo(1));
    CounterUpdate resultCounter = metricsCounterUpdates.iterator().next();
    assertTrue(new CounterHamcrestMatchers.CounterUpdateIntegerValueMatcher(expectedCounterValue).matches(resultCounter));
    assertEquals("ExpectedCounter", resultCounter.getStructuredNameAndMetadata().getName().getName());
}

Example 5 with InstructionRequestHandler

use of org.apache.beam.runners.fnexecution.control.InstructionRequestHandler in project beam by apache.

the class RegisterAndProcessBundleOperationTest method testGetProcessBundleProgressFetchesProgressResponseWhenBundleIdCached.

@Test
public void testGetProcessBundleProgressFetchesProgressResponseWhenBundleIdCached() throws Exception {
    InstructionRequestHandler mockInstructionRequestHandler = mock(InstructionRequestHandler.class);
    RegisterAndProcessBundleOperation operation = new RegisterAndProcessBundleOperation(IdGenerators.decrementingLongs(), mockInstructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    // this generates and caches bundleId
    operation.getProcessBundleInstructionId();
    ProcessBundleProgressResponse expectedResult = ProcessBundleProgressResponse.newBuilder().build();
    InstructionResponse instructionResponse = InstructionResponse.newBuilder().setProcessBundleProgress(expectedResult).build();
    CompletableFuture resultFuture = CompletableFuture.completedFuture(instructionResponse);
    when(mockInstructionRequestHandler.handle(any())).thenReturn(resultFuture);
    final ProcessBundleProgressResponse result = MoreFutures.get(operation.getProcessBundleProgress());
    assertSame("Return value from mockInstructionRequestHandler", expectedResult, result);
}