Examples with KinesisInputProperties org.talend.components.kinesis.input.KinesisInputProperties used on opensource projects

Example 1 with KinesisInputProperties

use of org.talend.components.kinesis.input.KinesisInputProperties in project components by Talend.

the class KinesisDatasetRuntime method getSample.

@Override
public void getSample(int limit, Consumer<IndexedRecord> consumer) {
    // Create an input runtime based on the properties.
    KinesisInputRuntime inputRuntime = new KinesisInputRuntime();
    KinesisInputProperties inputProperties = new KinesisInputProperties(null);
    inputProperties.init();
    inputProperties.setDatasetProperties(properties);
    inputProperties.useMaxNumRecords.setValue(true);
    inputProperties.maxNumRecords.setValue(limit);
    inputProperties.useMaxReadTime.setValue(true);
    inputProperties.maxReadTime.setValue(10000l);
    inputProperties.position.setValue(KinesisInputProperties.OffsetType.EARLIEST);
    inputRuntime.initialize(null, inputProperties);
    // Create a pipeline using the input component to get records.
    DirectOptions options = BeamLocalRunnerOption.getOptions();
    final Pipeline p = Pipeline.create(options);
    try (DirectConsumerCollector<IndexedRecord> collector = DirectConsumerCollector.of(consumer)) {
        // Collect a sample of the input records.
        p.apply(// 
        inputRuntime).apply(Sample.<IndexedRecord>any(limit)).apply(collector);
        p.run().waitUntilFinish();
    }
}

Example 2 with KinesisInputProperties

use of org.talend.components.kinesis.input.KinesisInputProperties in project components by Talend.

the class KinesisTestConstants method getInputFromBeginning.

public static KinesisInputProperties getInputFromBeginning(KinesisDatasetProperties dataset, Long maxReadTime, Integer maxNumRecords) {
    KinesisInputProperties input = new KinesisInputProperties("kinesisInput");
    input.init();
    input.setDatasetProperties(dataset);
    input.position.setValue(KinesisInputProperties.OffsetType.EARLIEST);
    if (maxReadTime != null) {
        input.useMaxReadTime.setValue(true);
        input.maxReadTime.setValue(maxReadTime);
    }
    if (maxNumRecords != null) {
        input.useMaxNumRecords.setValue(true);
        input.maxNumRecords.setValue(maxNumRecords);
    }
    return input;
}

Example 3 with KinesisInputProperties

use of org.talend.components.kinesis.input.KinesisInputProperties in project components by Talend.

the class KinesisInputRuntimeTestIT method testRuntime.

@Test
public void testRuntime() {
    KinesisInputProperties props = new KinesisInputProperties("KinesisInput");
    RuntimeInfo ri = def.getRuntimeInfo(ExecutionEngine.BEAM, props, ConnectorTopology.OUTGOING);
    try (SandboxedInstance si = RuntimeUtil.createRuntimeClass(ri, getClass().getClassLoader())) {
        Assert.assertEquals("KinesisInputRuntime", si.getInstance().getClass().getSimpleName());
    }
}