Examples with KV org.apache.samza.operators.KV used on opensource projects

Example 56 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class AsyncStreamTaskIntegrationTest method testAsyncTaskWithMultiplePartitionMultithreaded.

@Test
public void testAsyncTaskWithMultiplePartitionMultithreaded() throws Exception {
    Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
    Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
    genData(inputPartitionData, expectedOutputPartitionData);
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("ints", new NoOpSerde<>());
    InMemoryOutputDescriptor imod = isd.getOutputDescriptor("ints-out", new NoOpSerde<>());
    TestRunner.of(MyAsyncStreamTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).addConfig("task.max.concurrency", "4").run(Duration.ofSeconds(2));
    StreamAssert.containsInAnyOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}

Example 57 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class AsyncStreamTaskIntegrationTest method testAsyncTaskWithMultiplePartition.

@Test
public void testAsyncTaskWithMultiplePartition() throws Exception {
    Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
    Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
    genData(inputPartitionData, expectedOutputPartitionData);
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("ints", new NoOpSerde<KV>());
    InMemoryOutputDescriptor imod = isd.getOutputDescriptor("ints-out", new NoOpSerde<>());
    TestRunner.of(MyAsyncStreamTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).run(Duration.ofSeconds(2));
    StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}

Example 58 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class OperatorImplGraph method getOrCreatePartialJoinOpImpls.

private PartialJoinOperatorImpl getOrCreatePartialJoinOpImpls(JoinOperatorSpec joinOpSpec, boolean isLeft, Clock clock) {
    // get the per task pair of PartialJoinOperatorImpl for the corresponding {@code joinOpSpec}
    KV<PartialJoinOperatorImpl, PartialJoinOperatorImpl> partialJoinOpImpls = joinOpImpls.computeIfAbsent(joinOpSpec.getOpId(), joinOpId -> {
        PartialJoinFunction leftJoinFn = createLeftJoinFn(joinOpSpec);
        PartialJoinFunction rightJoinFn = createRightJoinFn(joinOpSpec);
        return new KV(new PartialJoinOperatorImpl(joinOpSpec, true, leftJoinFn, rightJoinFn, clock), new PartialJoinOperatorImpl(joinOpSpec, false, rightJoinFn, leftJoinFn, clock));
    });
    if (isLeft) {
        // we got here from the left side of the join
        return partialJoinOpImpls.getKey();
    } else {
        // we got here from the right side of the join
        return partialJoinOpImpls.getValue();
    }
}

Example 59 with KV

use of org.apache.samza.operators.KV in project beam by apache.

the class ImpulseTranslator method translate.

@Override
public void translate(PTransform<PBegin, PCollection<byte[]>> transform, Node node, TranslationContext ctx) {
    final PCollection<byte[]> output = ctx.getOutput(transform);
    final String outputId = ctx.getIdForPValue(output);
    final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(outputId, SamzaImpulseSystemFactory.class.getName());
    // The KvCoder is needed here for Samza not to crop the key.
    final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor = systemDescriptor.getInputDescriptor(outputId, kvSerde);
    ctx.registerInputMessageStream(output, inputDescriptor);
}

Example 60 with KV

use of org.apache.samza.operators.KV in project beam by apache.

the class ImpulseTranslator method translatePortable.

@Override
public void translatePortable(PipelineNode.PTransformNode transform, QueryablePipeline pipeline, PortableTranslationContext ctx) {
    final String outputId = ctx.getOutputId(transform);
    final String escapedOutputId = SamzaPipelineTranslatorUtils.escape(outputId);
    final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(escapedOutputId, SamzaImpulseSystemFactory.class.getName());
    // The KvCoder is needed here for Samza not to crop the key.
    final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
    final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor = systemDescriptor.getInputDescriptor(escapedOutputId, kvSerde);
    ctx.registerInputMessageStream(outputId, inputDescriptor);
}