Examples with ImmutableList org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList used on opensource projects

Example 1 with ImmutableList

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList in project beam by apache.

the class DockerEnvironmentFactory method createEnvironment.

/**
 * Creates a new, active {@link RemoteEnvironment} backed by a local Docker container.
 */
@Override
public RemoteEnvironment createEnvironment(Environment environment, String workerId) throws Exception {
    Preconditions.checkState(environment.getUrn().equals(BeamUrns.getUrn(RunnerApi.StandardEnvironments.Environments.DOCKER)), "The passed environment does not contain a DockerPayload.");
    final RunnerApi.DockerPayload dockerPayload = RunnerApi.DockerPayload.parseFrom(environment.getPayload());
    // Prepare docker invocation.
    String containerImage = dockerPayload.getContainerImage();
    // TODO: https://issues.apache.org/jira/browse/BEAM-4148 The default service address will not
    // work for Docker for Mac.
    String provisionEndpoint = provisioningServiceServer.getApiServiceDescriptor().getUrl();
    ImmutableList.Builder<String> dockerOptsBuilder = ImmutableList.<String>builder().addAll(gcsCredentialArgs()).add("--network=host").add("--env=DOCKER_MAC_CONTAINER=" + System.getenv("DOCKER_MAC_CONTAINER"));
    final boolean retainDockerContainer = pipelineOptions.as(ManualDockerEnvironmentOptions.class).getRetainDockerContainers();
    String semiPersistDir = pipelineOptions.as(RemoteEnvironmentOptions.class).getSemiPersistDir();
    ImmutableList.Builder<String> argsBuilder = ImmutableList.<String>builder().add(String.format("--id=%s", workerId)).add(String.format("--provision_endpoint=%s", provisionEndpoint));
    if (semiPersistDir != null) {
        argsBuilder.add(String.format("--semi_persist_dir=%s", semiPersistDir));
    }
    LOG.debug("Creating Docker Container with ID {}", workerId);
    // Wrap the blocking call to clientSource.get in case an exception is thrown.
    String containerId = null;
    InstructionRequestHandler instructionHandler = null;
    try {
        containerId = docker.runImage(containerImage, dockerOptsBuilder.build(), argsBuilder.build());
        LOG.debug("Created Docker Container with Container ID {}", containerId);
        // Wait on a client from the gRPC server.
        while (instructionHandler == null) {
            try {
                // If the docker is not alive anymore, we abort.
                if (!docker.isContainerRunning(containerId)) {
                    IllegalStateException illegalStateException = new IllegalStateException(String.format("No container running for id %s", containerId));
                    try {
                        String containerLogs = docker.getContainerLogs(containerId);
                        LOG.error("Docker container {} logs:\n{}", containerId, containerLogs);
                    } catch (Exception getLogsException) {
                        illegalStateException.addSuppressed(getLogsException);
                    }
                    throw illegalStateException;
                }
                instructionHandler = clientSource.take(workerId, Duration.ofSeconds(5));
            } catch (TimeoutException timeoutEx) {
                LOG.info("Still waiting for startup of environment {} for worker id {}", dockerPayload.getContainerImage(), workerId);
            } catch (InterruptedException interruptEx) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(interruptEx);
            }
        }
    } catch (Exception e) {
        if (containerId != null) {
            // Kill the launched docker container if we can't retrieve a client for it.
            try {
                docker.killContainer(containerId);
                if (!retainDockerContainer) {
                    docker.removeContainer(containerId);
                }
            } catch (Exception dockerException) {
                e.addSuppressed(dockerException);
            }
        }
        throw e;
    }
    return DockerContainerEnvironment.create(docker, environment, containerId, instructionHandler, retainDockerContainer);
}

Example 2 with ImmutableList

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList in project beam by apache.

the class ProcessEnvironmentFactory method createEnvironment.

/**
 * Creates a new, active {@link RemoteEnvironment} backed by a forked process.
 */
@Override
public RemoteEnvironment createEnvironment(Environment environment, String workerId) throws Exception {
    Preconditions.checkState(environment.getUrn().equals(BeamUrns.getUrn(RunnerApi.StandardEnvironments.Environments.PROCESS)), "The passed environment does not contain a ProcessPayload.");
    final RunnerApi.ProcessPayload processPayload = RunnerApi.ProcessPayload.parseFrom(environment.getPayload());
    String executable = processPayload.getCommand();
    String provisionEndpoint = provisioningServiceServer.getApiServiceDescriptor().getUrl();
    String semiPersistDir = pipelineOptions.as(RemoteEnvironmentOptions.class).getSemiPersistDir();
    ImmutableList.Builder<String> argsBuilder = ImmutableList.<String>builder().add(String.format("--id=%s", workerId)).add(String.format("--provision_endpoint=%s", provisionEndpoint));
    if (semiPersistDir != null) {
        argsBuilder.add(String.format("--semi_persist_dir=%s", semiPersistDir));
    }
    LOG.debug("Creating Process for worker ID {}", workerId);
    // Wrap the blocking call to clientSource.get in case an exception is thrown.
    InstructionRequestHandler instructionHandler = null;
    try {
        ProcessManager.RunningProcess process = processManager.startProcess(workerId, executable, argsBuilder.build(), processPayload.getEnvMap());
        // Wait on a client from the gRPC server.
        while (instructionHandler == null) {
            try {
                // If the process is not alive anymore, we abort.
                process.isAliveOrThrow();
                instructionHandler = clientSource.take(workerId, Duration.ofSeconds(5));
            } catch (TimeoutException timeoutEx) {
                LOG.info("Still waiting for startup of environment '{}' for worker id {}", processPayload.getCommand(), workerId);
            } catch (InterruptedException interruptEx) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(interruptEx);
            }
        }
    } catch (Exception e) {
        try {
            processManager.stopProcess(workerId);
        } catch (Exception processKillException) {
            e.addSuppressed(processKillException);
        }
        throw e;
    }
    return ProcessEnvironment.create(processManager, environment, workerId, instructionHandler);
}

Example 3 with ImmutableList

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList in project beam by apache.

the class DefaultJobBundleFactory method createEnvironmentCaches.

private ImmutableList<EnvironmentCacheAndLock> createEnvironmentCaches(ThrowingFunction<ServerFactory, ServerInfo> serverInfoCreator, int count) {
    ImmutableList.Builder<EnvironmentCacheAndLock> caches = ImmutableList.builder();
    for (int i = 0; i < count; i++) {
        final Lock refLock;
        if (environmentExpirationMillis > 0) {
            // The lock ensures there is no race condition between expiring an environment and a client
            // still attempting to use it, hence referencing it.
            refLock = new ReentrantLock(true);
        } else {
            refLock = NoopLock.get();
        }
        CacheBuilder<Environment, WrappedSdkHarnessClient> cacheBuilder = CacheBuilder.newBuilder().removalListener(notification -> {
            WrappedSdkHarnessClient client = notification.getValue();
            final int refCount;
            // We need to use a lock here to ensure we are not causing the environment to
            // be removed if beforehand a StageBundleFactory has retrieved it but not yet
            // issued ref() on it.
            refLock.lock();
            try {
                refCount = client.unref();
            } finally {
                refLock.unlock();
            }
            if (refCount > 0) {
                LOG.warn("Expiring environment {} with {} remaining bundle references. Taking note to clean it up during shutdown if the references are not removed by then.", notification.getKey(), refCount);
                evictedActiveClients.add(client);
            }
        });
        if (environmentExpirationMillis > 0) {
            cacheBuilder.expireAfterWrite(environmentExpirationMillis, TimeUnit.MILLISECONDS);
        }
        LoadingCache<Environment, WrappedSdkHarnessClient> cache = cacheBuilder.build(new CacheLoader<Environment, WrappedSdkHarnessClient>() {

            @Override
            public WrappedSdkHarnessClient load(Environment environment) throws Exception {
                EnvironmentFactory.Provider environmentFactoryProvider = environmentFactoryProviderMap.get(environment.getUrn());
                ServerFactory serverFactory = environmentFactoryProvider.getServerFactory();
                ServerInfo serverInfo = serverInfoCreator.apply(serverFactory);
                String workerId = stageIdGenerator.getId();
                serverInfo.getProvisioningServer().getService().registerEnvironment(workerId, environment);
                EnvironmentFactory environmentFactory = environmentFactoryProvider.createEnvironmentFactory(serverInfo.getControlServer(), serverInfo.getLoggingServer(), serverInfo.getRetrievalServer(), serverInfo.getProvisioningServer(), clientPool, stageIdGenerator);
                return WrappedSdkHarnessClient.wrapping(environmentFactory.createEnvironment(environment, workerId), serverInfo);
            }
        });
        caches.add(new EnvironmentCacheAndLock(cache, refLock));
    }
    return caches.build();
}

Example 4 with ImmutableList

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList in project beam by apache.

the class FlinkPipelineExecutionEnvironmentTest method shouldUseStreamingTransformOverridesWithUnboundedSources.

@Test
public void shouldUseStreamingTransformOverridesWithUnboundedSources() {
    FlinkPipelineOptions options = FlinkPipelineOptions.defaults();
    // no explicit streaming mode set
    options.setRunner(FlinkRunner.class);
    FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options);
    Pipeline p = Mockito.spy(Pipeline.create(options));
    // Add unbounded source which will set the streaming mode to true
    p.apply(GenerateSequence.from(0));
    flinkEnv.translate(p);
    ArgumentCaptor<ImmutableList> captor = ArgumentCaptor.forClass(ImmutableList.class);
    Mockito.verify(p).replaceAll(captor.capture());
    ImmutableList<PTransformOverride> overridesList = captor.getValue();
    assertThat(overridesList, hasItem(PTransformOverride.of(PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), CreateStreamingFlinkView.Factory.INSTANCE)));
}

Example 5 with ImmutableList

use of org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList in project beam by apache.

the class JetMetricsContainer method extractUpdates.

private <UpdateT, CellT extends AbstractMetric<UpdateT>> ImmutableList<MetricUpdates.MetricUpdate<UpdateT>> extractUpdates(Map<MetricName, CellT> cells) {
    ImmutableList.Builder<MetricUpdates.MetricUpdate<UpdateT>> updates = ImmutableList.builder();
    for (CellT cell : cells.values()) {
        UpdateT value = cell.getValue();
        if (value != null) {
            MetricKey key = MetricKey.create(stepName, cell.getName());
            MetricUpdates.MetricUpdate<UpdateT> update = MetricUpdates.MetricUpdate.create(key, value);
            updates.add(update);
        }
    }
    return updates.build();
}