Search in sources :

Example 1 with Sample

use of org.talend.sdk.component.runtime.beam.data.Sample in project component-runtime by Talend.

the class BeamIOWrappingTest method mapper.

@Test
public void mapper() {
    final Object source = newComponent("beamio_input", ComponentManager.ComponentType.MAPPER);
    assertThat(source, instanceOf(BeamSource.class));
    final Mapper mapper = new BeamMapperImpl((PTransform<PBegin, ?>) source, getPlugin(), "test", "beamio_input");
    mapper.start();
    assertEquals(2, mapper.assess());
    final Input input = mapper.create();
    assertNotNull(input);
    input.start();
    assertEquals(new Sample("a"), input.next());
    assertEquals(new Sample("b"), input.next());
    assertNull(input.next());
    input.stop();
    mapper.stop();
}
Also used : PartitionMapper(org.talend.sdk.component.api.input.PartitionMapper) Mapper(org.talend.sdk.component.runtime.input.Mapper) Input(org.talend.sdk.component.runtime.input.Input) BeamMapperImpl(org.talend.sdk.component.runtime.beam.impl.BeamMapperImpl) Sample(org.talend.sdk.component.runtime.beam.data.Sample) PBegin(org.apache.beam.sdk.values.PBegin) Test(org.junit.Test)

Example 2 with Sample

use of org.talend.sdk.component.runtime.beam.data.Sample in project component-runtime by Talend.

the class BeamComponentExtensionTest method assertMapper.

private void assertMapper(final Mapper mapper) {
    assertNotNull(mapper);
    mapper.start();
    final Input input = mapper.create();
    input.start();
    assertEquals(new Sample("a"), input.next());
    assertEquals(new Sample("b"), input.next());
    assertNull(input.next());
    input.stop();
    mapper.stop();
}
Also used : Input(org.talend.sdk.component.runtime.input.Input) Sample(org.talend.sdk.component.runtime.beam.data.Sample)

Example 3 with Sample

use of org.talend.sdk.component.runtime.beam.data.Sample in project component-runtime by Talend.

the class BeamIOWrappingTest method outputChain.

@Test
public void outputChain() {
    MySink.DATA.clear();
    final Object source = newComponent("beamio_output_chain", ComponentManager.ComponentType.PROCESSOR);
    final Processor processor = new BeamProcessorChainImpl((PTransform<PCollection<?>, PDone>) source, null, getPlugin(), "test", "beamio_output");
    processor.start();
    processor.beforeGroup();
    Stream.of("tsrif", "dnoces").forEach(data -> processor.onNext(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return new Sample(data);
    }, name -> value -> MySink.DATA.add(value.toString())));
    processor.afterGroup(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return value -> MySink.DATA.add(value.toString());
    });
    processor.stop();
    assertEquals(asList("setup", "start-bundle", "first", "second", "finish-out", "finish-bundle", "teardown"), MySink.DATA);
    MySink.DATA.clear();
}
Also used : PartitionMapper(org.talend.sdk.component.api.input.PartitionMapper) PBegin(org.apache.beam.sdk.values.PBegin) PipelineResult(org.apache.beam.sdk.PipelineResult) RequiredArgsConstructor(lombok.RequiredArgsConstructor) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) Assert.assertThat(org.junit.Assert.assertThat) Sample(org.talend.sdk.component.runtime.beam.data.Sample) Create(org.apache.beam.sdk.transforms.Create) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) Assert.fail(org.junit.Assert.fail) ClassRule(org.junit.ClassRule) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) Processor(org.talend.sdk.component.runtime.output.Processor) Collectors.joining(java.util.stream.Collectors.joining) StandardCharsets(java.nio.charset.StandardCharsets) Serializable(java.io.Serializable) List(java.util.List) Branches(org.talend.sdk.component.runtime.output.Branches) Stream(java.util.stream.Stream) ParDo(org.apache.beam.sdk.transforms.ParDo) Optional(java.util.Optional) BeamMapperImpl(org.talend.sdk.component.runtime.beam.impl.BeamMapperImpl) BeamProcessorChainImpl(org.talend.sdk.component.runtime.beam.impl.BeamProcessorChainImpl) Getter(lombok.Getter) CapturingPipeline(org.talend.sdk.component.runtime.beam.impl.CapturingPipeline) Coder(org.apache.beam.sdk.coders.Coder) HashMap(java.util.HashMap) Option(org.talend.sdk.component.api.configuration.Option) AtomicReference(java.util.concurrent.atomic.AtomicReference) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) FileBasedSink(org.apache.beam.sdk.io.FileBasedSink) TestPipeline(org.apache.beam.sdk.testing.TestPipeline) DelegatingTransform(org.talend.sdk.component.runtime.beam.transform.DelegatingTransform) Input(org.talend.sdk.component.runtime.input.Input) OutputStream(java.io.OutputStream) DoFn(org.apache.beam.sdk.transforms.DoFn) PDone(org.apache.beam.sdk.values.PDone) Files(java.nio.file.Files) PAssert(org.apache.beam.sdk.testing.PAssert) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) Test(org.junit.Test) PCollection(org.apache.beam.sdk.values.PCollection) InputStreamReader(java.io.InputStreamReader) File(java.io.File) Collectors.toList(java.util.stream.Collectors.toList) Mapper(org.talend.sdk.component.runtime.input.Mapper) Assert.assertNull(org.junit.Assert.assertNull) Rule(org.junit.Rule) Instant(org.joda.time.Instant) JarLocation.jarLocation(org.apache.ziplock.JarLocation.jarLocation) BufferedReader(java.io.BufferedReader) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) Assert.assertEquals(org.junit.Assert.assertEquals) TextIO(org.apache.beam.sdk.io.TextIO) TemporaryFolder(org.junit.rules.TemporaryFolder) InputStream(java.io.InputStream) PCollection(org.apache.beam.sdk.values.PCollection) Processor(org.talend.sdk.component.runtime.output.Processor) BeamProcessorChainImpl(org.talend.sdk.component.runtime.beam.impl.BeamProcessorChainImpl) PDone(org.apache.beam.sdk.values.PDone) Sample(org.talend.sdk.component.runtime.beam.data.Sample) Test(org.junit.Test)

Aggregations

PBegin (org.apache.beam.sdk.values.PBegin)2 Sample (org.talend.sdk.component.runtime.beam.data.Sample)2 Input (org.talend.sdk.component.runtime.input.Input)2 BufferedReader (java.io.BufferedReader)1 File (java.io.File)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 InputStreamReader (java.io.InputStreamReader)1 OutputStream (java.io.OutputStream)1 Serializable (java.io.Serializable)1 StandardCharsets (java.nio.charset.StandardCharsets)1 Files (java.nio.file.Files)1 ArrayList (java.util.ArrayList)1 Arrays.asList (java.util.Arrays.asList)1 Collection (java.util.Collection)1 Collections.emptyList (java.util.Collections.emptyList)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1