Examples with CoderRegistry org.apache.beam.sdk.coders.CoderRegistry used on opensource projects

Example 16 with CoderRegistry

use of org.apache.beam.sdk.coders.CoderRegistry in project beam by apache.

the class KafkaIOTest method testInferKeyCoderFailure.

@Test
public void testInferKeyCoderFailure() throws Exception {
    cannotInferException.expect(RuntimeException.class);
    CoderRegistry registry = CoderRegistry.createDefault();
    KafkaIO.inferCoder(registry, NonInferableObjectDeserializer.class);
}

Example 17 with CoderRegistry

use of org.apache.beam.sdk.coders.CoderRegistry in project beam by apache.

the class LocalDeserializerProviderTest method testInferKeyCoder.

@Test
public void testInferKeyCoder() {
    CoderRegistry registry = CoderRegistry.createDefault();
    assertTrue(LocalDeserializerProvider.of(LongDeserializer.class).getCoder(registry).getValueCoder() instanceof VarLongCoder);
    assertTrue(LocalDeserializerProvider.of(StringDeserializer.class).getCoder(registry).getValueCoder() instanceof StringUtf8Coder);
    assertTrue(LocalDeserializerProvider.of(InstantDeserializer.class).getCoder(registry).getValueCoder() instanceof InstantCoder);
    assertTrue(LocalDeserializerProvider.of(DeserializerWithInterfaces.class).getCoder(registry).getValueCoder() instanceof VarLongCoder);
}

Example 18 with CoderRegistry

use of org.apache.beam.sdk.coders.CoderRegistry in project beam by apache.

the class DataTokenizationTest method fileSystemIORead.

private PCollection<Row> fileSystemIORead(String inputGcsFilePattern, FORMAT inputGcsFileFormat) throws IOException {
    DataTokenizationOptions options = PipelineOptionsFactory.create().as(DataTokenizationOptions.class);
    options.setDataSchemaPath(SCHEMA_FILE_PATH);
    options.setInputFilePattern(inputGcsFilePattern);
    options.setInputFileFormat(inputGcsFileFormat);
    if (inputGcsFileFormat == FORMAT.CSV) {
        options.setCsvContainsHeaders(Boolean.FALSE);
    }
    SchemasUtils testSchemaUtils = new SchemasUtils(options.getDataSchemaPath(), StandardCharsets.UTF_8);
    CoderRegistry coderRegistry = testPipeline.getCoderRegistry();
    coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    coderRegistry.registerCoderForType(RowCoder.of(testSchemaUtils.getBeamSchema()).getEncodedTypeDescriptor(), RowCoder.of(testSchemaUtils.getBeamSchema()));
    /*
     * Row/Row Coder for FailsafeElement.
     */
    FailsafeElementCoder<Row, Row> coder = FailsafeElementCoder.of(RowCoder.of(testSchemaUtils.getBeamSchema()), RowCoder.of(testSchemaUtils.getBeamSchema()));
    coderRegistry.registerCoderForType(coder.getEncodedTypeDescriptor(), coder);
    return new TokenizationFileSystemIO(options).read(testPipeline, testSchemaUtils);
}

Example 19 with CoderRegistry

use of org.apache.beam.sdk.coders.CoderRegistry in project beam by apache.

the class ReduceFnTester method combining.

/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy} and {@link CombineFn},
 * creating a {@link TriggerStateMachine} from the {@link Trigger} in the {@link
 * WindowingStrategy}.
 */
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining(WindowingStrategy<?, W> strategy, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception {
    CoderRegistry registry = CoderRegistry.createDefault();
    // Ensure that the CombineFn can be converted into an AppliedCombineFn
    AppliedCombineFn.withInputCoder(combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
    return combining(strategy, TriggerStateMachines.stateMachineForTrigger(TriggerTranslation.toProto(strategy.getTrigger())), combineFn, outputCoder);
}

Example 20 with CoderRegistry

use of org.apache.beam.sdk.coders.CoderRegistry in project beam by apache.

the class ReduceFnTester method combining.

/**
 * Creates a {@link ReduceFnTester} for the given {@link WindowingStrategy}, {@link CombineFn},
 * and {@link TriggerStateMachine}, for mocking the interaction between {@link ReduceFnRunner} and
 * the {@link TriggerStateMachine}. Ignores the {@link Trigger} in the {@link WindowingStrategy}.
 */
public static <W extends BoundedWindow, AccumT, OutputT> ReduceFnTester<Integer, OutputT, W> combining(WindowingStrategy<?, W> strategy, TriggerStateMachine triggerStateMachine, CombineFn<Integer, AccumT, OutputT> combineFn, Coder<OutputT> outputCoder) throws Exception {
    CoderRegistry registry = CoderRegistry.createDefault();
    AppliedCombineFn<String, Integer, AccumT, OutputT> fn = AppliedCombineFn.withInputCoder(combineFn, registry, KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()));
    return new ReduceFnTester<>(strategy, triggerStateMachine, SystemReduceFn.combining(StringUtf8Coder.of(), fn), outputCoder, PipelineOptionsFactory.create(), NullSideInputReader.empty());
}