Examples with Processor org.talend.sdk.component.runtime.output.Processor used on opensource projects

Example 16 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamMapperImpl method createFlowDefinition.

private static FlowDefinition createFlowDefinition(final PTransform<PBegin, ?> begin, final String plugin, final String family, final String name) {
    final CapturingPipeline capturingPipeline = new CapturingPipeline(PipelineOptionsFactory.create());
    final CapturingPipeline.SourceExtractor sourceExtractor = new CapturingPipeline.SourceExtractor();
    capturingPipeline.apply(begin);
    capturingPipeline.traverseTopologically(sourceExtractor);
    PTransform<? super PBegin, ?> transform = sourceExtractor.getTransform();
    if (transform == null) {
        capturingPipeline.apply(begin);
        transform = capturingPipeline.getRoot();
    }
    if (Read.Bounded.class.isInstance(transform)) {
        final Processor processor = sourceExtractor.getTransforms().isEmpty() ? null : new BeamProcessorChainImpl(sourceExtractor.getTransforms(), capturingPipeline.getCoderRegistry(), plugin, family, name);
        return new FlowDefinition(Read.Bounded.class.cast(transform).getSource(), processor);
    }
    if (Read.Unbounded.class.isInstance(transform)) {
        final Processor processor = sourceExtractor.getTransforms().isEmpty() ? null : new BeamProcessorChainImpl(sourceExtractor.getTransforms(), capturingPipeline.getCoderRegistry(), plugin, family, name);
        return new FlowDefinition(Read.Unbounded.class.cast(transform).getSource(), processor);
    }
    throw new IllegalArgumentException("This implementation only supports Bounded sources");
}

Example 17 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamProcessorChainImpl method beforeGroup.

@Override
public void beforeGroup() {
    beforeChunkCount = 0;
    for (final Processor p : processors) {
        p.beforeGroup();
        beforeChunkCount++;
    }
}

Example 18 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class ProcessorTest method processor.

@Test
public void processor() {
    final Processor processor = COMPONENT_FACTORY.createProcessor(SampleProcessor.class, new Object());
    final JoinInputFactory joinInputFactory = new JoinInputFactory().withInput("__default__", asList(new SampleProcessor.Sample(1), Json.createObjectBuilder().add("data", 2).build()));
    final PCollection<JsonObject> inputs = pipeline.apply(Data.of(processor.plugin(), joinInputFactory.asInputRecords()));
    final PCollection<Map<String, JsonObject>> outputs = inputs.apply(TalendFn.asFn(processor)).apply(Data.map(processor.plugin(), JsonObject.class));
    PAssert.that(outputs).satisfies((SerializableFunction<Iterable<Map<String, JsonObject>>, Void>) input -> {
        final List<Map<String, JsonObject>> result = StreamSupport.stream(input.spliterator(), false).collect(toList());
        assertEquals(2, result.size());
        result.forEach(e -> assertTrue(e.containsKey("__default__") && e.containsKey("reject")));
        assertEquals(new HashSet<>(asList(1, 2)), result.stream().map(e -> e.get("__default__").getInt("data")).collect(toSet()));
        return null;
    });
    assertEquals(PipelineResult.State.DONE, pipeline.run().waitUntilFinish());
}

Example 19 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class TInOutputBeamTest method processor.

@Test
@Ignore("You need to complete this test with your own data and assertions")
public void processor() {
    // Output configuration
    // Setup your component configuration for the test here
    final TInOutputConfiguration configuration = new TInOutputConfiguration();
    // We create the component processor instance using the configuration filled above
    final Processor processor = COMPONENT_FACTORY.createProcessor(TInOutput.class, configuration);
    // The join input factory construct inputs test data for every input branch you have defined for this component
    // Make sure to fil in some test data for the branches you want to test
    // You can also remove the branches that you don't need from the factory below
    final JoinInputFactory joinInputFactory = new JoinInputFactory().withInput("__default__", asList());
    // Convert it to a beam "source"
    final PCollection<JsonObject> inputs = pipeline.apply(Data.of(processor.plugin(), joinInputFactory.asInputRecords()));
    // add our processor right after to see each data as configured previously
    inputs.apply(TalendFn.asFn(processor)).apply(Data.map(processor.plugin(), JsonObject.class));
    // run the pipeline and ensure the execution was successful
    assertEquals(PipelineResult.State.DONE, pipeline.run().waitUntilFinish());
}

Example 20 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class TInProcessorBeamTest method processor.

@Test
@Ignore("You need to complete this test with your own data and assertions")
public void processor() {
    // Processor configuration
    // Setup your component configuration for the test here
    final TInProcessorConfiguration configuration = new TInProcessorConfiguration();
    // We create the component processor instance using the configuration filled above
    final Processor processor = COMPONENT_FACTORY.createProcessor(TInProcessor.class, configuration);
    // The join input factory construct inputs test data for every input branch you have defined for this component
    // Make sure to fil in some test data for the branches you want to test
    // You can also remove the branches that you don't need from the factory below
    final JoinInputFactory joinInputFactory = new JoinInputFactory().withInput("__default__", asList());
    // Convert it to a beam "source"
    final PCollection<JsonObject> inputs = pipeline.apply(Data.of(processor.plugin(), joinInputFactory.asInputRecords()));
    // add our processor right after to see each data as configured previously
    final PCollection<Map<String, JsonObject>> outputs = inputs.apply(TalendFn.asFn(processor)).apply(Data.map(processor.plugin(), JsonObject.class));
    PAssert.that(outputs).satisfies((SerializableFunction<Iterable<Map<String, JsonObject>>, Void>) input -> {
        final List<Map<String, JsonObject>> result = StreamSupport.stream(input.spliterator(), false).collect(toList());
        return null;
    });
    // run the pipeline and ensure the execution was successful
    assertEquals(PipelineResult.State.DONE, pipeline.run().waitUntilFinish());
}