Examples with Coder org.apache.beam.sdk.coders.Coder used on opensource projects

Example 66 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class PTransformRunnerFactoryTestContext method addOutgoingDataEndpoint.

@Override
public <T> FnDataReceiver<T> addOutgoingDataEndpoint(ApiServiceDescriptor apiServiceDescriptor, Coder<T> coder) {
    BeamFnDataOutboundAggregator aggregator = getOutboundAggregators().get(apiServiceDescriptor);
    FnDataReceiver<T> receiver = aggregator.registerOutputDataLocation(getPTransformId(), coder);
    getOutgoingDataEndpoints().computeIfAbsent(apiServiceDescriptor, (unused) -> new ArrayList<>()).add(DataEndpoint.create(getPTransformId(), coder, receiver));
    return receiver;
}

Example 67 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class BeamFnDataOutboundAggregatorTest method testConfiguredTimeLimit.

@Test
public void testConfiguredTimeLimit() throws Exception {
    List<Elements> values = new ArrayList<>();
    PipelineOptions options = PipelineOptionsFactory.create();
    options.as(ExperimentalOptions.class).setExperiments(Arrays.asList("data_buffer_time_limit_ms=1"));
    final CountDownLatch waitForFlush = new CountDownLatch(1);
    BeamFnDataOutboundAggregator aggregator = new BeamFnDataOutboundAggregator(options, endpoint::getInstructionId, TestStreams.withOnNext((Consumer<Elements>) e -> {
        values.add(e);
        waitForFlush.countDown();
    }).build(), false);
    // Test that it emits when time passed the time limit
    FnDataReceiver<byte[]> dataReceiver = registerOutputLocation(aggregator, endpoint, CODER);
    aggregator.start();
    dataReceiver.accept(new byte[1]);
    // wait the flush thread to flush the buffer
    waitForFlush.await();
    assertEquals(messageWithData(new byte[1]), values.get(0));
}

Example 68 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class DynamicProtoCoderTest method testDynamicMessage.

@Test
public void testDynamicMessage() throws Exception {
    DynamicMessage message = DynamicMessage.newBuilder(MessageA.getDescriptor()).setField(MessageA.getDescriptor().findFieldByNumber(MessageA.FIELD1_FIELD_NUMBER), "foo").build();
    Coder<DynamicMessage> coder = DynamicProtoCoder.of(message.getDescriptorForType());
    // Special code to check the DynamicMessage equality (@see IsDynamicMessageEqual)
    for (Coder.Context context : ALL_CONTEXTS) {
        CoderProperties.coderDecodeEncodeInContext(coder, context, message, IsDynamicMessageEqual.equalTo(message));
    }
}

Example 69 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class JmsIOTest method testCheckpointMarkDefaultCoder.

/**
 * Test the checkpoint mark default coder, which is actually AvroCoder.
 */
@Test
public void testCheckpointMarkDefaultCoder() throws Exception {
    JmsCheckpointMark jmsCheckpointMark = new JmsCheckpointMark();
    jmsCheckpointMark.add(new ActiveMQMessage());
    Coder coder = new JmsIO.UnboundedJmsSource(null).getCheckpointMarkCoder();
    CoderProperties.coderSerializable(coder);
    CoderProperties.coderDecodeEncodeEqual(coder, jmsCheckpointMark);
}

Example 70 with Coder

use of org.apache.beam.sdk.coders.Coder in project beam by apache.

the class ProcessBundleBenchmark method testTinyBundle.

@Benchmark
// Use several threads since we expect contention during bundle processing.
@Threads(16)
public void testTinyBundle(TrivialTransform trivialTransform) throws Exception {
    Map<String, ? super Coder<WindowedValue<?>>> remoteOutputCoders = trivialTransform.descriptor.getRemoteOutputCoders();
    Map<String, RemoteOutputReceiver<?>> outputReceivers = new HashMap<>();
    AtomicInteger outputValuesCount = new AtomicInteger();
    for (Entry<String, ? super Coder<WindowedValue<?>>> remoteOutputCoder : remoteOutputCoders.entrySet()) {
        outputReceivers.put(remoteOutputCoder.getKey(), RemoteOutputReceiver.of((Coder) remoteOutputCoder.getValue(), (FnDataReceiver<? super WindowedValue<?>>) (WindowedValue<?> value) -> outputValuesCount.incrementAndGet()));
    }
    try (RemoteBundle bundle = trivialTransform.processor.newBundle(outputReceivers, BundleProgressHandler.ignored())) {
        Iterables.getOnlyElement(bundle.getInputReceivers().values()).accept(valueInGlobalWindow(new byte[0]));
    }
    assertEquals(3, outputValuesCount.getAndSet(0));
}