Search in sources :

Example 6 with ComponentManager

use of org.talend.sdk.component.runtime.manager.ComponentManager in project component-runtime by Talend.

the class DIBatchSimulationTest method toBeam.

@Test
void toBeam() {
    final ComponentManager manager = ComponentManager.instance();
    final Collection<Object> sourceData = new ArrayList<>();
    final Collection<Object> processorData = new ArrayList<>();
    To.RECORDS.clear();
    doDi(manager, sourceData, processorData, manager.findProcessor("DIBatchSimulationTest", "to", 1, new HashMap<>()).map(p -> {
        assertTrue(QueueOutput.class.isInstance(p), p.getClass().getName());
        return p;
    }), manager.findMapper("DIBatchSimulationTest", "passthroughinput", 1, singletonMap("count", "1000")));
    assertEquals(1000, sourceData.size());
    assertEquals(1000, processorData.size());
    assertEquals(1000, To.RECORDS.size());
}
Also used : Assertions.fail(org.junit.jupiter.api.Assertions.fail) IntStream(java.util.stream.IntStream) PartitionMapper(org.talend.sdk.component.api.input.PartitionMapper) Producer(org.talend.sdk.component.api.input.Producer) SerializableCoder(org.apache.beam.sdk.coders.SerializableCoder) PBegin(org.apache.beam.sdk.values.PBegin) AutoChunkProcessor(org.talend.sdk.component.runtime.di.AutoChunkProcessor) Getter(lombok.Getter) HashMap(java.util.HashMap) Option(org.talend.sdk.component.api.configuration.Option) OutputFactory(org.talend.sdk.component.runtime.output.OutputFactory) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) Create(org.apache.beam.sdk.transforms.Create) ElementListener(org.talend.sdk.component.api.processor.ElementListener) InputFactory(org.talend.sdk.component.runtime.output.InputFactory) Map(java.util.Map) ToString(lombok.ToString) Collections.singletonMap(java.util.Collections.singletonMap) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Input(org.talend.sdk.component.runtime.input.Input) JobStateAware(org.talend.sdk.component.runtime.di.JobStateAware) DoFn(org.apache.beam.sdk.transforms.DoFn) Emitter(org.talend.sdk.component.api.input.Emitter) JsonObject(javax.json.JsonObject) PDone(org.apache.beam.sdk.values.PDone) Collection(java.util.Collection) OutputsHandler(org.talend.sdk.component.runtime.di.OutputsHandler) PCollection(org.apache.beam.sdk.values.PCollection) ChainedMapper(org.talend.sdk.component.runtime.manager.chain.ChainedMapper) Processor(org.talend.sdk.component.runtime.output.Processor) Serializable(java.io.Serializable) PrimitiveIterator(java.util.PrimitiveIterator) Test(org.junit.jupiter.api.Test) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Mapper(org.talend.sdk.component.runtime.input.Mapper) Branches(org.talend.sdk.component.runtime.output.Branches) InputsHandler(org.talend.sdk.component.runtime.di.InputsHandler) ParDo(org.apache.beam.sdk.transforms.ParDo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Data(lombok.Data) Optional(java.util.Optional) Jsonb(javax.json.bind.Jsonb) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) AllArgsConstructor(lombok.AllArgsConstructor) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) NoArgsConstructor(lombok.NoArgsConstructor) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) ArrayList(java.util.ArrayList) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) JsonObject(javax.json.JsonObject) Test(org.junit.jupiter.api.Test)

Example 7 with ComponentManager

use of org.talend.sdk.component.runtime.manager.ComponentManager in project component-runtime by Talend.

the class DIBatchSimulationTest method fromBeam.

@Test
void fromBeam() {
    final ComponentManager manager = ComponentManager.instance();
    final Collection<Object> sourceData = new ArrayList<>();
    final Collection<Object> processorData = new ArrayList<>();
    doDi(manager, sourceData, processorData, manager.findProcessor("DIBatchSimulationTest", "passthroughoutput", 1, new HashMap<>()), manager.findMapper("DIBatchSimulationTest", "from", 1, singletonMap("count", "1000")).map(m -> {
        assertTrue(QueueMapper.class.isInstance(m), m.getClass().getName());
        return m;
    }));
    assertEquals(1000, sourceData.size());
    assertEquals(1000, processorData.size());
}
Also used : Assertions.fail(org.junit.jupiter.api.Assertions.fail) IntStream(java.util.stream.IntStream) PartitionMapper(org.talend.sdk.component.api.input.PartitionMapper) Producer(org.talend.sdk.component.api.input.Producer) SerializableCoder(org.apache.beam.sdk.coders.SerializableCoder) PBegin(org.apache.beam.sdk.values.PBegin) AutoChunkProcessor(org.talend.sdk.component.runtime.di.AutoChunkProcessor) Getter(lombok.Getter) HashMap(java.util.HashMap) Option(org.talend.sdk.component.api.configuration.Option) OutputFactory(org.talend.sdk.component.runtime.output.OutputFactory) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) Create(org.apache.beam.sdk.transforms.Create) ElementListener(org.talend.sdk.component.api.processor.ElementListener) InputFactory(org.talend.sdk.component.runtime.output.InputFactory) Map(java.util.Map) ToString(lombok.ToString) Collections.singletonMap(java.util.Collections.singletonMap) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Input(org.talend.sdk.component.runtime.input.Input) JobStateAware(org.talend.sdk.component.runtime.di.JobStateAware) DoFn(org.apache.beam.sdk.transforms.DoFn) Emitter(org.talend.sdk.component.api.input.Emitter) JsonObject(javax.json.JsonObject) PDone(org.apache.beam.sdk.values.PDone) Collection(java.util.Collection) OutputsHandler(org.talend.sdk.component.runtime.di.OutputsHandler) PCollection(org.apache.beam.sdk.values.PCollection) ChainedMapper(org.talend.sdk.component.runtime.manager.chain.ChainedMapper) Processor(org.talend.sdk.component.runtime.output.Processor) Serializable(java.io.Serializable) PrimitiveIterator(java.util.PrimitiveIterator) Test(org.junit.jupiter.api.Test) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Mapper(org.talend.sdk.component.runtime.input.Mapper) Branches(org.talend.sdk.component.runtime.output.Branches) InputsHandler(org.talend.sdk.component.runtime.di.InputsHandler) ParDo(org.apache.beam.sdk.transforms.ParDo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Data(lombok.Data) Optional(java.util.Optional) Jsonb(javax.json.bind.Jsonb) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) AllArgsConstructor(lombok.AllArgsConstructor) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) NoArgsConstructor(lombok.NoArgsConstructor) HashMap(java.util.HashMap) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) ArrayList(java.util.ArrayList) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) JsonObject(javax.json.JsonObject) Test(org.junit.jupiter.api.Test)

Example 8 with ComponentManager

use of org.talend.sdk.component.runtime.manager.ComponentManager in project component-runtime by Talend.

the class DIBatchSimulationTest method emptyToBeam.

@Test
void emptyToBeam() {
    final ComponentManager manager = ComponentManager.instance();
    final Collection<Object> sourceData = new ArrayList<>();
    doDi(manager, sourceData, new ArrayList<>(), manager.findProcessor("DIBatchSimulationTest", "to", 1, new HashMap<>()).map(p -> {
        assertTrue(QueueOutput.class.isInstance(p), p.getClass().getName());
        return p;
    }), manager.findMapper("DIBatchSimulationTest", "passthroughinput", 1, singletonMap("count", "0")));
    assertEquals(0, sourceData.size());
}
Also used : Assertions.fail(org.junit.jupiter.api.Assertions.fail) IntStream(java.util.stream.IntStream) PartitionMapper(org.talend.sdk.component.api.input.PartitionMapper) Producer(org.talend.sdk.component.api.input.Producer) SerializableCoder(org.apache.beam.sdk.coders.SerializableCoder) PBegin(org.apache.beam.sdk.values.PBegin) AutoChunkProcessor(org.talend.sdk.component.runtime.di.AutoChunkProcessor) Getter(lombok.Getter) HashMap(java.util.HashMap) Option(org.talend.sdk.component.api.configuration.Option) OutputFactory(org.talend.sdk.component.runtime.output.OutputFactory) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) Create(org.apache.beam.sdk.transforms.Create) ElementListener(org.talend.sdk.component.api.processor.ElementListener) InputFactory(org.talend.sdk.component.runtime.output.InputFactory) Map(java.util.Map) ToString(lombok.ToString) Collections.singletonMap(java.util.Collections.singletonMap) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) Input(org.talend.sdk.component.runtime.input.Input) JobStateAware(org.talend.sdk.component.runtime.di.JobStateAware) DoFn(org.apache.beam.sdk.transforms.DoFn) Emitter(org.talend.sdk.component.api.input.Emitter) JsonObject(javax.json.JsonObject) PDone(org.apache.beam.sdk.values.PDone) Collection(java.util.Collection) OutputsHandler(org.talend.sdk.component.runtime.di.OutputsHandler) PCollection(org.apache.beam.sdk.values.PCollection) ChainedMapper(org.talend.sdk.component.runtime.manager.chain.ChainedMapper) Processor(org.talend.sdk.component.runtime.output.Processor) Serializable(java.io.Serializable) PrimitiveIterator(java.util.PrimitiveIterator) Test(org.junit.jupiter.api.Test) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Mapper(org.talend.sdk.component.runtime.input.Mapper) Branches(org.talend.sdk.component.runtime.output.Branches) InputsHandler(org.talend.sdk.component.runtime.di.InputsHandler) ParDo(org.apache.beam.sdk.transforms.ParDo) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Data(lombok.Data) Optional(java.util.Optional) Jsonb(javax.json.bind.Jsonb) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) AllArgsConstructor(lombok.AllArgsConstructor) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) NoArgsConstructor(lombok.NoArgsConstructor) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) ArrayList(java.util.ArrayList) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) JsonObject(javax.json.JsonObject) Test(org.junit.jupiter.api.Test)

Example 9 with ComponentManager

use of org.talend.sdk.component.runtime.manager.ComponentManager in project component-runtime by Talend.

the class AutoKVWrapperTest method test.

@Test
public void test() {
    final ComponentManager instance = ComponentManager.instance();
    final JsonBuilderFactory factory = instance.getJsonpBuilderFactory();
    PAssert.that(buildBaseJsonPipeline(pipeline, factory).setCoder(JsonpJsonObjectCoder.of(null)).apply(AutoKVWrapper.of(null, JobImpl.LocalSequenceHolder.cleanAndGet(getClass().getName() + ".test"), "", ""))).satisfies(values -> {
        final List<KV<String, JsonObject>> items = StreamSupport.stream(values.spliterator(), false).sorted(comparing(k -> k.getValue().getJsonArray("b1").getJsonObject(0).getString("foo"))).collect(toList());
        assertEquals(2, items.size());
        // ensure we got 2 ids
        assertEquals(2, new HashSet<>(items).size());
        assertEquals(asList("a", "b"), items.stream().map(k -> k.getValue().getJsonArray("b1").getJsonObject(0).getString("foo")).collect(toList()));
        return null;
    });
    assertEquals(PipelineResult.State.DONE, pipeline.run().waitUntilFinish());
}
Also used : JsonBuilderFactory(javax.json.JsonBuilderFactory) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) KV(org.apache.beam.sdk.values.KV) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 10 with ComponentManager

use of org.talend.sdk.component.runtime.manager.ComponentManager in project component-runtime by Talend.

the class RepositoryModelBuilderTest method test.

@Test
void test(final TemporaryFolder temporaryFolder, final TestInfo testInfo) throws Exception {
    final String pluginName = testInfo.getTestMethod().get().getName() + ".jar";
    final File pluginJar = createChainPlugin(temporaryFolder.getRoot(), pluginName);
    try (final ComponentManager manager = new ComponentManager(new File("target/fake-m2"), "TALEND-INF/dependencies.txt", null)) {
        manager.addPlugin(pluginJar.getAbsolutePath());
        Container pluginContainer = manager.findPlugin(pluginName).orElseThrow(() -> new Exception("test plugin don't exist"));
        assertNotNull(pluginContainer);
        RepositoryModel rm = pluginContainer.get(RepositoryModel.class);
        assertNotNull(rm);
        assertEquals(1, rm.getFamilies().size());
        Family family = rm.getFamilies().get(0);
        String ds1Id = IdGenerator.get("family1", "datastore", "dataStore1");
        Config dataStore1Config = family.getConfigs().stream().filter(c -> c.getId().equals(ds1Id)).findFirst().get();
        assertNotNull(dataStore1Config);
        assertEquals(1, dataStore1Config.getChildConfigs().size());
        assertEquals("configuration1", dataStore1Config.getChildConfigs().get(0).getMeta().getName());
        String ds2Id = IdGenerator.get("family1", "datastore", "dataStore2");
        Config dataStore2Config = family.getConfigs().stream().filter(c -> c.getId().equals(ds2Id)).findFirst().get();
        assertNotNull(dataStore2Config);
        assertEquals(1, dataStore2Config.getChildConfigs().size());
        assertEquals("configuration2", dataStore2Config.getChildConfigs().get(0).getMeta().getName());
    }
}
Also used : Container(org.talend.sdk.component.container.Container) ComponentManager(org.talend.sdk.component.runtime.manager.ComponentManager) RepositoryModel(org.talend.sdk.component.design.extension.RepositoryModel) File(java.io.File) IOException(java.io.IOException) Test(org.junit.jupiter.api.Test)

Aggregations

ComponentManager (org.talend.sdk.component.runtime.manager.ComponentManager)21 Test (org.junit.jupiter.api.Test)12 File (java.io.File)8 JsonObject (javax.json.JsonObject)8 List (java.util.List)7 Collectors.toList (java.util.stream.Collectors.toList)7 Collection (java.util.Collection)6 HashMap (java.util.HashMap)6 Jsonb (javax.json.bind.Jsonb)6 AllArgsConstructor (lombok.AllArgsConstructor)6 Data (lombok.Data)6 Assertions.assertEquals (org.junit.jupiter.api.Assertions.assertEquals)6 Serializable (java.io.Serializable)5 ArrayList (java.util.ArrayList)5 Collections.singletonMap (java.util.Collections.singletonMap)5 Map (java.util.Map)5 Optional (java.util.Optional)5 PrimitiveIterator (java.util.PrimitiveIterator)5 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)5 IntStream (java.util.stream.IntStream)5