Examples with Processor org.talend.sdk.component.runtime.output.Processor used on opensource projects

Example 11 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamExecutor method run.

@Override
public void run() {
    try {
        final Map<String, Mapper> mappers = delegate.getLevels().values().stream().flatMap(Collection::stream).filter(Job.Component::isSource).collect(toMap(Job.Component::getId, e -> delegate.getManager().findMapper(e.getNode().getFamily(), e.getNode().getComponent(), e.getNode().getVersion(), e.getNode().getConfiguration()).orElseThrow(() -> new IllegalStateException("No mapper found for: " + e.getNode()))));
        final Map<String, Processor> processors = delegate.getLevels().values().stream().flatMap(Collection::stream).filter(component -> !component.isSource()).collect(toMap(Job.Component::getId, e -> delegate.getManager().findProcessor(e.getNode().getFamily(), e.getNode().getComponent(), e.getNode().getVersion(), e.getNode().getConfiguration()).orElseThrow(() -> new IllegalStateException("No processor found for:" + e.getNode()))));
        final Pipeline pipeline = Pipeline.create(createPipelineOptions());
        final Map<String, PCollection<JsonObject>> pCollections = new HashMap<>();
        delegate.getLevels().values().stream().flatMap(Collection::stream).forEach(component -> {
            if (component.isSource()) {
                final Mapper mapper = mappers.get(component.getId());
                pCollections.put(component.getId(), pipeline.apply(toName("TalendIO", component), TalendIO.read(mapper)).apply(toName("RecordNormalizer", component), RecordNormalizer.of(mapper.plugin())));
            } else {
                final Processor processor = processors.get(component.getId());
                final List<Job.Edge> joins = getEdges(delegate.getEdges(), component, e -> e.getTo().getNode());
                final Map<String, PCollection<KV<String, JsonObject>>> inputs = joins.stream().collect(toMap(e -> e.getTo().getBranch(), e -> {
                    final PCollection<JsonObject> pc = pCollections.get(e.getFrom().getNode().getId());
                    final PCollection<JsonObject> filteredInput = pc.apply(toName("RecordBranchFilter", component, e), RecordBranchFilter.of(processor.plugin(), e.getFrom().getBranch()));
                    final PCollection<JsonObject> mappedInput;
                    if (e.getFrom().getBranch().equals(e.getTo().getBranch())) {
                        mappedInput = filteredInput;
                    } else {
                        mappedInput = filteredInput.apply(toName("RecordBranchMapper", component, e), RecordBranchMapper.of(processor.plugin(), e.getFrom().getBranch(), e.getTo().getBranch()));
                    }
                    return mappedInput.apply(toName("RecordBranchUnwrapper", component, e), RecordBranchUnwrapper.of(processor.plugin(), e.getTo().getBranch())).apply(toName("AutoKVWrapper", component, e), AutoKVWrapper.of(processor.plugin(), delegate.getKeyProvider(component.getId()), component.getId(), e.getFrom().getBranch()));
                }));
                KeyedPCollectionTuple<String> join = null;
                for (final Map.Entry<String, PCollection<KV<String, JsonObject>>> entry : inputs.entrySet()) {
                    final TupleTag<JsonObject> branch = new TupleTag<>(entry.getKey());
                    join = join == null ? KeyedPCollectionTuple.of(branch, entry.getValue()) : join.and(branch, entry.getValue());
                }
                final PCollection<JsonObject> preparedInput = join.apply(toName("CoGroupByKey", component), CoGroupByKey.create()).apply(toName("CoGroupByKeyResultMappingTransform", component), new CoGroupByKeyResultMappingTransform<>(processor.plugin(), true));
                if (getEdges(delegate.getEdges(), component, e -> e.getFrom().getNode()).isEmpty()) {
                    final PTransform<PCollection<JsonObject>, PDone> write = TalendIO.write(processor);
                    preparedInput.apply(toName("Output", component), write);
                } else {
                    final PTransform<PCollection<JsonObject>, PCollection<JsonObject>> process = TalendFn.asFn(processor);
                    pCollections.put(component.getId(), preparedInput.apply(toName("Processor", component), process));
                }
            }
        });
        final PipelineResult result = pipeline.run();
        // the wait until finish don't wait for the job to complete on the direct runner
        result.waitUntilFinish();
        while (PipelineResult.State.RUNNING.equals(result.getState())) {
            try {
                Thread.sleep(100L);
            } catch (final InterruptedException e) {
                throw new IllegalStateException("the job was aborted", e);
            }
        }
    } finally {
        delegate.getLevels().values().stream().flatMap(Collection::stream).map(Job.Component::getId).forEach(JobImpl.LocalSequenceHolder::clean);
    }
}

Example 12 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamIOWrappingTest method processor.

@Test
public void processor() {
    MySink.DATA.clear();
    final Object source = newComponent("beamio_output", ComponentManager.ComponentType.PROCESSOR);
    final Processor processor = new BeamProcessorChainImpl((PTransform<PCollection<?>, ?>) source, null, getPlugin(), "test", "beamio_output");
    processor.start();
    processor.beforeGroup();
    Stream.of("tsrif", "dnoces").forEach(data -> processor.onNext(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return data;
    }, null));
    processor.afterGroup(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return value -> MySink.DATA.add(value.toString());
    });
    processor.stop();
    assertEquals(asList("setup", "start-bundle", "first", "second", "finish-out", "finish-bundle", "teardown"), MySink.DATA);
    MySink.DATA.clear();
}

Example 13 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamIOWrappingTest method outputChain.

@Test
public void outputChain() {
    MySink.DATA.clear();
    final Object source = newComponent("beamio_output_chain", ComponentManager.ComponentType.PROCESSOR);
    final Processor processor = new BeamProcessorChainImpl((PTransform<PCollection<?>, PDone>) source, null, getPlugin(), "test", "beamio_output");
    processor.start();
    processor.beforeGroup();
    Stream.of("tsrif", "dnoces").forEach(data -> processor.onNext(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return new Sample(data);
    }, name -> value -> MySink.DATA.add(value.toString())));
    processor.afterGroup(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return value -> MySink.DATA.add(value.toString());
    });
    processor.stop();
    assertEquals(asList("setup", "start-bundle", "first", "second", "finish-out", "finish-bundle", "teardown"), MySink.DATA);
    MySink.DATA.clear();
}

Example 14 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamIOWrappingTest method fileOutput.

@Test
public void fileOutput() throws IOException {
    final Object source = newComponent("beamio_text", ComponentManager.ComponentType.PROCESSOR);
    final Processor processor = new BeamProcessorChainImpl((PTransform<PCollection<?>, PDone>) source, null, getPlugin(), "test", "beamio_text");
    processor.start();
    processor.beforeGroup();
    Stream.of("first", "second").forEach(data -> processor.onNext(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return data;
    }, name -> value -> fail(name + " >> " + value)));
    final AtomicReference<Object> output = new AtomicReference<>();
    processor.afterGroup(name -> {
        assertEquals(Branches.DEFAULT_BRANCH, name);
        return output::set;
    });
    processor.stop();
    final FileBasedSink.FileResult result = FileBasedSink.FileResult.class.cast(output.get());
    assertNotNull(result);
    final File file = new File(result.getTempFilename().toString());
    assertTrue(file.exists());
    assertEquals(file.getParentFile().getParentFile(), TEMPORARY_FOLDER.getRoot());
    assertEquals("first\nsecond", Files.lines(file.toPath()).collect(joining("\n")));
}

Example 15 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class DIBatchSimulationTest method doDi.

private void doDi(final ComponentManager manager, final Collection<Object> sourceData, final Collection<Object> processorData, final Optional<Processor> proc, final Optional<Mapper> mapper) {
    final Map<String, Object> globalMap = new HashMap<>();
    try {
        final Processor processor = proc.orElseThrow(() -> new IllegalStateException("scanning failed"));
        JobStateAware.init(processor, globalMap);
        final Jsonb jsonbProcessor = Jsonb.class.cast(manager.findPlugin(processor.plugin()).get().get(ComponentManager.AllServices.class).getServices().get(Jsonb.class));
        final AutoChunkProcessor processorProcessor = new AutoChunkProcessor(100, processor);
        processorProcessor.start();
        globalMap.put("processorProcessor", processorProcessor);
        final InputsHandler inputsHandlerProcessor = new InputsHandler(jsonbProcessor);
        inputsHandlerProcessor.addConnection("FLOW", row1Struct.class);
        final OutputsHandler outputHandlerProcessor = new OutputsHandler(jsonbProcessor);
        final InputFactory inputsProcessor = inputsHandlerProcessor.asInputFactory();
        final OutputFactory outputsProcessor = outputHandlerProcessor.asOutputFactory();
        final Mapper tempMapperMapper = mapper.orElseThrow(() -> new IllegalStateException("scanning failed"));
        JobStateAware.init(tempMapperMapper, globalMap);
        doRun(manager, sourceData, processorData, globalMap, processorProcessor, inputsHandlerProcessor, outputHandlerProcessor, inputsProcessor, outputsProcessor, tempMapperMapper);
    } finally {
        doClose(globalMap);
    }
}