Examples with ApiServiceDescriptor org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor used on opensource projects

Example 1 with ApiServiceDescriptor

use of org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor in project beam by apache.

the class ServerFactoryTest method testCreatingUnixDomainSocketServer.

@Test
public void testCreatingUnixDomainSocketServer() throws Exception {
    assumeTrue(Epoll.isAvailable());
    Endpoints.ApiServiceDescriptor apiServiceDescriptor = runTestUsing(ServerFactory.createEpollDomainSocket(), ManagedChannelFactory.createEpoll());
    assertThat(apiServiceDescriptor.getUrl(), startsWith("unix://" + System.getProperty("java.io.tmpdir")));
}

Example 2 with ApiServiceDescriptor

use of org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor in project beam by apache.

the class ServerFactoryTest method testCreatingEpollServer.

@Test
public void testCreatingEpollServer() throws Exception {
    assumeTrue(Epoll.isAvailable());
    // tcnative only supports the ipv4 address family
    assumeTrue(InetAddress.getLoopbackAddress() instanceof Inet4Address);
    Endpoints.ApiServiceDescriptor apiServiceDescriptor = runTestUsing(ServerFactory.createEpollSocket(), ManagedChannelFactory.createEpoll());
    HostAndPort hostAndPort = HostAndPort.fromString(apiServiceDescriptor.getUrl());
    assertThat(hostAndPort.getHost(), anyOf(equalTo(InetAddress.getLoopbackAddress().getHostName()), equalTo(InetAddress.getLoopbackAddress().getHostAddress())));
    assertThat(hostAndPort.getPort(), allOf(greaterThan(0), lessThan(65536)));
}

Example 3 with ApiServiceDescriptor

use of org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor in project beam by apache.

the class ServerFactoryTest method defaultServerWorks.

@Test
public void defaultServerWorks() throws Exception {
    Endpoints.ApiServiceDescriptor apiServiceDescriptor = runTestUsing(ServerFactory.createDefault(), ManagedChannelFactory.createDefault());
    HostAndPort hostAndPort = HostAndPort.fromString(apiServiceDescriptor.getUrl());
    assertThat(hostAndPort.getHost(), anyOf(equalTo(InetAddress.getLoopbackAddress().getHostName()), equalTo(InetAddress.getLoopbackAddress().getHostAddress())));
    assertThat(hostAndPort.getPort(), allOf(greaterThan(0), lessThan(65536)));
}

Example 4 with ApiServiceDescriptor

use of org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor in project beam by apache.

the class ProcessBundleDescriptors method fromExecutableStageInternal.

private static ExecutableProcessBundleDescriptor fromExecutableStageInternal(String id, ExecutableStage stage, ApiServiceDescriptor dataEndpoint, @Nullable ApiServiceDescriptor stateEndpoint) throws IOException {
    // Create with all of the processing transforms, and all of the components.
    // TODO: Remove the unreachable subcomponents if the size of the descriptor matters.
    Map<String, PTransform> stageTransforms = stage.getTransforms().stream().collect(Collectors.toMap(PTransformNode::getId, PTransformNode::getTransform));
    Components.Builder components = stage.getComponents().toBuilder().clearTransforms().putAllTransforms(stageTransforms);
    ImmutableList.Builder<RemoteInputDestination> inputDestinationsBuilder = ImmutableList.builder();
    ImmutableMap.Builder<String, Coder> remoteOutputCodersBuilder = ImmutableMap.builder();
    WireCoderSetting wireCoderSetting = stage.getWireCoderSettings().stream().filter(ws -> ws.getInputOrOutputId().equals(stage.getInputPCollection().getId())).findAny().orElse(WireCoderSetting.getDefaultInstance());
    // The order of these does not matter.
    inputDestinationsBuilder.add(addStageInput(dataEndpoint, stage.getInputPCollection(), components, wireCoderSetting));
    remoteOutputCodersBuilder.putAll(addStageOutputs(dataEndpoint, stage.getOutputPCollections(), components, stage.getWireCoderSettings()));
    Map<String, Map<String, SideInputSpec>> sideInputSpecs = addSideInputs(stage, components);
    Map<String, Map<String, BagUserStateSpec>> bagUserStateSpecs = forBagUserStates(stage, components.build());
    Map<String, Map<String, TimerSpec>> timerSpecs = forTimerSpecs(stage, components);
    lengthPrefixAnyInputCoder(stage.getInputPCollection().getId(), components);
    // Copy data from components to ProcessBundleDescriptor.
    ProcessBundleDescriptor.Builder bundleDescriptorBuilder = ProcessBundleDescriptor.newBuilder().setId(id);
    if (stateEndpoint != null) {
        bundleDescriptorBuilder.setStateApiServiceDescriptor(stateEndpoint);
    }
    if (timerSpecs.size() > 0) {
        // By default use the data endpoint for timers, in the future considering enabling specifying
        // a different ApiServiceDescriptor for timers.
        bundleDescriptorBuilder.setTimerApiServiceDescriptor(dataEndpoint);
    }
    bundleDescriptorBuilder.putAllCoders(components.getCodersMap()).putAllEnvironments(components.getEnvironmentsMap()).putAllPcollections(components.getPcollectionsMap()).putAllWindowingStrategies(components.getWindowingStrategiesMap()).putAllTransforms(components.getTransformsMap());
    return ExecutableProcessBundleDescriptor.of(bundleDescriptorBuilder.build(), inputDestinationsBuilder.build(), remoteOutputCodersBuilder.build(), sideInputSpecs, bagUserStateSpecs, timerSpecs);
}

Example 5 with ApiServiceDescriptor

use of org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor in project beam by apache.

the class ProcessBundleDescriptors method addStageOutputs.

private static Map<String, Coder<WindowedValue<?>>> addStageOutputs(ApiServiceDescriptor dataEndpoint, Collection<PCollectionNode> outputPCollections, Components.Builder components, Collection<WireCoderSetting> wireCoderSettings) throws IOException {
    Map<String, Coder<WindowedValue<?>>> remoteOutputCoders = new LinkedHashMap<>();
    for (PCollectionNode outputPCollection : outputPCollections) {
        WireCoderSetting wireCoderSetting = wireCoderSettings.stream().filter(ws -> ws.getInputOrOutputId().equals(outputPCollection.getId())).findAny().orElse(WireCoderSetting.getDefaultInstance());
        OutputEncoding outputEncoding = addStageOutput(dataEndpoint, components, outputPCollection, wireCoderSetting);
        remoteOutputCoders.put(outputEncoding.getPTransformId(), outputEncoding.getCoder());
    }
    return remoteOutputCoders;
}