Search in sources :

Example 1 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class ComponentExtensionTest method processorCollector.

@Test
void processorCollector() {
    final Processor processor = handler.createProcessor(Transform.class, null);
    final SimpleComponentRule.Outputs outputs = handler.collect(processor, new JoinInputFactory().withInput("__default__", asList(new Transform.Record("a"), new Transform.Record("bb"))).withInput("second", asList(new Transform.Record("1"), new Transform.Record("2"))));
    assertEquals(2, outputs.size());
    assertEquals(asList(2, 3), outputs.get(Integer.class, "size"));
    assertEquals(asList("a1", "bb2"), outputs.get(String.class, "value"));
}
Also used : Processor(org.talend.sdk.component.runtime.output.Processor) JoinInputFactory(org.talend.sdk.component.junit.JoinInputFactory) SimpleComponentRule(org.talend.sdk.component.junit.SimpleComponentRule) Transform(org.talend.sdk.component.junit.component.Transform) Test(org.junit.jupiter.api.Test)

Example 2 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamProcessorChainImpl method onNext.

@Override
public void onNext(final InputFactory input, final OutputFactory output) {
    Collection<InputFactory> finalInput = singletonList(input);
    if (!preProcessors.isEmpty()) {
        for (final Processor p : preProcessors) {
            final StoringOuputFactory tmpOutput = new StoringOuputFactory();
            finalInput.forEach(in -> p.onNext(in, tmpOutput));
            if (tmpOutput.getValues() == null) {
                // chain is stopped
                return;
            }
            finalInput = tmpOutput.getValues().stream().map(val -> (InputFactory) name -> {
                if (!Branches.DEFAULT_BRANCH.equals(name)) {
                    throw new IllegalArgumentException("Only default branch is supported at the moment");
                }
                return val;
            }).collect(toList());
        }
    }
    finalInput.forEach(in -> lastProcessor.onNext(in, output));
}
Also used : CoderRegistry(org.apache.beam.sdk.coders.CoderRegistry) PBegin(org.apache.beam.sdk.values.PBegin) ObjectInputStream(java.io.ObjectInputStream) Coder(org.apache.beam.sdk.coders.Coder) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) OutputFactory(org.talend.sdk.component.runtime.output.OutputFactory) Collections.singletonList(java.util.Collections.singletonList) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) ByteArrayInputStream(java.io.ByteArrayInputStream) TupleTag(org.apache.beam.sdk.values.TupleTag) InputFactory(org.talend.sdk.component.runtime.output.InputFactory) Map(java.util.Map) PCollectionTuple(org.apache.beam.sdk.values.PCollectionTuple) InvalidObjectException(java.io.InvalidObjectException) OutputStream(java.io.OutputStream) DoFn(org.apache.beam.sdk.transforms.DoFn) ContainerFinder(org.talend.sdk.component.runtime.serialization.ContainerFinder) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) Delegated(org.talend.sdk.component.runtime.base.Delegated) IOException(java.io.IOException) PCollection(org.apache.beam.sdk.values.PCollection) Processor(org.talend.sdk.component.runtime.output.Processor) Serializer(org.talend.sdk.component.runtime.base.Serializer) EnhancedObjectInputStream(org.talend.sdk.component.runtime.serialization.EnhancedObjectInputStream) Serializable(java.io.Serializable) ObjectStreamException(java.io.ObjectStreamException) POutput(org.apache.beam.sdk.values.POutput) Lifecycle(org.talend.sdk.component.runtime.base.Lifecycle) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Branches(org.talend.sdk.component.runtime.output.Branches) Stream(java.util.stream.Stream) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) WindowingStrategy(org.apache.beam.sdk.values.WindowingStrategy) InputStream(java.io.InputStream) InputFactory(org.talend.sdk.component.runtime.output.InputFactory) Processor(org.talend.sdk.component.runtime.output.Processor)

Example 3 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class BeamProcessorChainImpl method start.

@Override
public void start() {
    startedCount = 0;
    for (final Processor p : processors) {
        p.start();
        startedCount++;
    }
}
Also used : Processor(org.talend.sdk.component.runtime.output.Processor)

Example 4 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class ComponentManagerTest method doCheckRegistry.

private void doCheckRegistry(final File plugin1, final File plugin2, final ComponentManager manager) throws Exception {
    Stream.of(plugin1, plugin2).map(File::getAbsolutePath).forEach(manager::addPlugin);
    final List<ContainerComponentRegistry> registries = manager.find(c -> Stream.of(c.get(ContainerComponentRegistry.class))).collect(toList());
    // we saw both plugin
    assertEquals(2, registries.size());
    registries.forEach(registry -> {
        final Container container = manager.find(c -> registry == c.get(ContainerComponentRegistry.class) ? Stream.of(c) : Stream.empty()).findFirst().get();
        assertEquals(1, registry.getServices().size());
        assertNotNull(registry.getServices().iterator().next().getInstance());
        final Collection<ServiceMeta.ActionMeta> actions = registry.getServices().iterator().next().getActions();
        assertEquals(1, actions.size());
        assertEquals(pluginGenerator.toPackage(container.getId()) + ".AModel", actions.iterator().next().getInvoker().apply(null).getClass().getName());
        assertEquals(1, registry.getComponents().size());
        registry.getComponents().forEach((name, component) -> {
            assertEquals("comp", name);
            assertEquals(name, component.getName());
            assertEquals(singletonList("Misc"), component.getCategories());
            assertEquals(1, component.getProcessors().size());
            component.getProcessors().forEach((procName, processorMeta) -> {
                assertEquals("proc", procName);
                assertEquals("default", processorMeta.getIcon());
                final String packageName = pluginGenerator.toPackage(container.getId());
                final Processor processor = processorMeta.getInstantiator().apply(emptyMap());
                assertNotNull(processor);
                final Object model;
                try {
                    final String className = packageName + ".AModel";
                    model = container.getLoader().loadClass(className).getConstructor().newInstance();
                } catch (final Exception e) {
                    fail(e.getMessage());
                    throw new IllegalArgumentException(e);
                }
                runProcessorLifecycle(model, processor);
            });
        });
    });
    // now try to execute the processor outside the correct TCCL and ensure it still
    // works
    final Container container = manager.find(Stream::of).findFirst().get();
    final String packageName = pluginGenerator.toPackage(container.getId());
    final ContainerComponentRegistry registry = container.get(ContainerComponentRegistry.class);
    final ComponentFamilyMeta componentFamilyMeta = registry.getComponents().values().iterator().next();
    final ComponentFamilyMeta.ProcessorMeta processorMeta = componentFamilyMeta.getProcessors().values().iterator().next();
    final Processor processor = processorMeta.getInstantiator().apply(emptyMap());
    final Object aModel = container.getLoader().loadClass(packageName + ".AModel").getConstructor().newInstance();
    runProcessorLifecycle(aModel, processor);
    // finally ensure it is serializable
    DynamicContainerFinder.LOADERS.clear();
    manager.find(Stream::of).forEach(c -> DynamicContainerFinder.LOADERS.put(c.getId(), c.getLoader()));
    runProcessorLifecycle(aModel, copy(processor, container.getLoader()));
}
Also used : Assertions.fail(org.junit.jupiter.api.Assertions.fail) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) AttributeNotFoundException(javax.management.AttributeNotFoundException) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Date(java.util.Date) PluginGenerator(org.talend.sdk.component.runtime.manager.asm.PluginGenerator) ObjectInputStream(java.io.ObjectInputStream) AtomicReference(java.util.concurrent.atomic.AtomicReference) Collections.singletonList(java.util.Collections.singletonList) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) ByteArrayInputStream(java.io.ByteArrayInputStream) WithTemporaryFolder(org.talend.sdk.component.junit.base.junit5.WithTemporaryFolder) MBeanServer(javax.management.MBeanServer) DynamicContainerFinder(org.talend.sdk.component.runtime.manager.serialization.DynamicContainerFinder) ObjectOutputStream(java.io.ObjectOutputStream) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ManagementFactory(java.lang.management.ManagementFactory) InstanceNotFoundException(javax.management.InstanceNotFoundException) ReflectionException(javax.management.ReflectionException) Collections.emptyMap(java.util.Collections.emptyMap) Container(org.talend.sdk.component.container.Container) Optional.ofNullable(java.util.Optional.ofNullable) Collection(java.util.Collection) IOException(java.io.IOException) UUID(java.util.UUID) ObjectName(javax.management.ObjectName) Assertions.assertNotSame(org.junit.jupiter.api.Assertions.assertNotSame) Processor(org.talend.sdk.component.runtime.output.Processor) EnhancedObjectInputStream(org.talend.sdk.component.runtime.serialization.EnhancedObjectInputStream) File(java.io.File) Test(org.junit.jupiter.api.Test) MBeanException(javax.management.MBeanException) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Stream(java.util.stream.Stream) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) TemporaryFolder(org.talend.sdk.component.junit.base.junit5.TemporaryFolder) Processor(org.talend.sdk.component.runtime.output.Processor) AttributeNotFoundException(javax.management.AttributeNotFoundException) InstanceNotFoundException(javax.management.InstanceNotFoundException) ReflectionException(javax.management.ReflectionException) IOException(java.io.IOException) MBeanException(javax.management.MBeanException) Container(org.talend.sdk.component.container.Container) ByteArrayOutputStream(java.io.ByteArrayOutputStream) ObjectInputStream(java.io.ObjectInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) ObjectOutputStream(java.io.ObjectOutputStream) EnhancedObjectInputStream(org.talend.sdk.component.runtime.serialization.EnhancedObjectInputStream) Stream(java.util.stream.Stream)

Example 5 with Processor

use of org.talend.sdk.component.runtime.output.Processor in project component-runtime by Talend.

the class AdvancedProcessorImplTest method subclassing.

@Test
void subclassing() {
    final Processor processor = new ProcessorImpl("Root", "Test", "Plugin", new SampleOutput());
    final AtomicReference<Object> ref = new AtomicReference<>();
    // just to enforce the init
    processor.beforeGroup();
    processor.onNext(name -> new Whatever(1), name -> value -> assertTrue(ref.compareAndSet(null, value)));
    final Object out = ref.get();
    assertNotNull(out);
    assertTrue(() -> String.class.isInstance(out));
    assertEquals("1", out.toString());
}
Also used : Processor(org.talend.sdk.component.runtime.output.Processor) ProcessorImpl(org.talend.sdk.component.runtime.output.ProcessorImpl) AtomicReference(java.util.concurrent.atomic.AtomicReference) Test(org.junit.jupiter.api.Test)

Aggregations

Processor (org.talend.sdk.component.runtime.output.Processor)20 List (java.util.List)8 Map (java.util.Map)8 Collectors.toList (java.util.stream.Collectors.toList)8 JsonObject (javax.json.JsonObject)8 Collection (java.util.Collection)7 Test (org.junit.Test)7 HashMap (java.util.HashMap)6 Stream (java.util.stream.Stream)6 PipelineResult (org.apache.beam.sdk.PipelineResult)6 PCollection (org.apache.beam.sdk.values.PCollection)6 IOException (java.io.IOException)5 Arrays.asList (java.util.Arrays.asList)5 AtomicReference (java.util.concurrent.atomic.AtomicReference)5 PAssert (org.apache.beam.sdk.testing.PAssert)5 TestPipeline (org.apache.beam.sdk.testing.TestPipeline)5 ClassRule (org.junit.ClassRule)5 Rule (org.junit.Rule)5 Mapper (org.talend.sdk.component.runtime.input.Mapper)5 BufferedReader (java.io.BufferedReader)4