Examples with KV org.apache.samza.operators.KV used on opensource projects

Example 1 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestEventHubsOutputDescriptor method testEntityConnectionConfigs.

@Test
public void testEntityConnectionConfigs() {
    String systemName = "eventHub";
    String streamId = "output-stream";
    EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName);
    EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor.getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()).withSasKeyName("secretkey").withSasKey("sasToken-123");
    Map<String, String> generatedConfigs = outputDescriptor.toConfig();
    assertEquals("eventHub", generatedConfigs.get("streams.output-stream.samza.system"));
    assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId)));
    assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId)));
    assertEquals("secretkey", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId)));
    assertEquals("sasToken-123", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId)));
}

Example 2 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestEventHubsInputDescriptor method testWithoutEntityConnectionConfigs.

@Test
public void testWithoutEntityConnectionConfigs() {
    String systemName = "eventHub";
    String streamId = "input-stream";
    EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName);
    EventHubsInputDescriptor<KV<String, String>> inputDescriptor = systemDescriptor.getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde());
    Map<String, String> generatedConfigs = inputDescriptor.toConfig();
    assertEquals("eventHub", generatedConfigs.get("streams.input-stream.samza.system"));
    assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId)));
    assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId)));
    assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId)));
    assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId)));
    assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId)));
    // verify that there are no other configs
    assertEquals(3, generatedConfigs.size());
}

Example 3 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class SendToTableWithUpdateOperatorImpl method handleMessageAsync.

@Override
protected CompletionStage<Collection<KV<K, UpdateMessage<U, V>>>> handleMessageAsync(KV<K, UpdateMessage<U, V>> message, MessageCollector collector, TaskCoordinator coordinator) {
    final UpdateOptions options = spec.getUpdateOptions();
    final CompletableFuture<Void> updateFuture = table.updateAsync(message.getKey(), message.getValue().getUpdate());
    return updateFuture.handle((result, ex) -> {
        if (ex == null) {
            // success, no need to Put a default value
            return false;
        } else if (ex.getCause() instanceof RecordNotFoundException && message.getValue().hasDefault()) {
            // to PUT a default record for the key and then apply the update.
            if (options == UpdateOptions.UPDATE_WITH_DEFAULTS) {
                return true;
            } else {
                throw new SamzaException("Put default failed for update as the Update options was set to " + options + ". Please use UpdateOptions.UPDATE_WITH_DEFAULTS instead.");
            }
        } else {
            throw new SamzaException("Update failed with exception: ", ex);
        }
    }).thenCompose(shouldPutDefault -> {
        if (shouldPutDefault) {
            final CompletableFuture<Void> putFuture = table.putAsync(message.getKey(), message.getValue().getDefault());
            return putFuture.exceptionally(ex -> {
                LOG.warn("PUT default failed due to an exception. Ignoring the exception and proceeding with update. " + "The exception encountered is: ", ex);
                return null;
            }).thenCompose(res -> table.updateAsync(message.getKey(), message.getValue().getUpdate())).exceptionally(ex -> {
                throw new SamzaException("Update after Put default failed with exception: ", ex);
            });
        } else {
            return CompletableFuture.completedFuture(null);
        }
    }).thenApply(result -> Collections.singleton(message));
}

Example 4 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithStreamTableJoinWithSideInputs.

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() {
    /**
     * Example stream-table join where table t is configured with input1 (64) as a side-input stream.
     *
     *                                   join-table t -> output1 (8)
     *                                        |
     *    input2 (16) -> partitionBy ("64") __|
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde())).withSideInputs(Arrays.asList("input1")).withSideInputsProcessor(mock(SideInputsProcessor.class));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream2.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").join(table, mock(StreamTableJoinFunction.class)).sendTo(output1);
    }, config);
}

Example 5 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestExecutionPlanner method testCreateJobGraphForTaskApplication.

@Test
public void testCreateJobGraphForTaskApplication() {
    TaskApplicationDescriptorImpl taskAppDesc = mock(TaskApplicationDescriptorImpl.class);
    // add interemediate streams
    String intermediateStream1 = "intermediate-stream1";
    String intermediateBroadcast = "intermediate-broadcast1";
    // intermediate stream1, not broadcast
    GenericInputDescriptor<KV<Object, Object>> intermediateInput1 = system1Descriptor.getInputDescriptor(intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde()));
    GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor(intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde()));
    // intermediate stream2, broadcast
    GenericInputDescriptor<KV<Object, Object>> intermediateBroacastInput1 = system1Descriptor.getInputDescriptor(intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>()));
    GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor(intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>()));
    inputDescriptors.put(intermediateStream1, intermediateInput1);
    outputDescriptors.put(intermediateStream1, intermediateOutput1);
    inputDescriptors.put(intermediateBroadcast, intermediateBroacastInput1);
    outputDescriptors.put(intermediateBroadcast, intermediateBroacastOutput1);
    Set<String> broadcastStreams = new HashSet<>();
    broadcastStreams.add(intermediateBroadcast);
    when(taskAppDesc.getInputDescriptors()).thenReturn(inputDescriptors);
    when(taskAppDesc.getInputStreamIds()).thenReturn(inputDescriptors.keySet());
    when(taskAppDesc.getOutputDescriptors()).thenReturn(outputDescriptors);
    when(taskAppDesc.getOutputStreamIds()).thenReturn(outputDescriptors.keySet());
    when(taskAppDesc.getTableDescriptors()).thenReturn(Collections.emptySet());
    when(taskAppDesc.getSystemDescriptors()).thenReturn(systemDescriptors);
    when(taskAppDesc.getIntermediateBroadcastStreamIds()).thenReturn(broadcastStreams);
    doReturn(MockTaskApplication.class).when(taskAppDesc).getAppClass();
    Map<String, String> systemStreamConfigs = new HashMap<>();
    inputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    outputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig()));
    systemDescriptors.forEach(sd -> systemStreamConfigs.putAll(sd.toConfig()));
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    JobGraph jobGraph = planner.createJobGraph(taskAppDesc);
    assertEquals(1, jobGraph.getJobNodes().size());
    assertTrue(jobGraph.getInputStreams().stream().map(edge -> edge.getName()).filter(streamId -> inputDescriptors.containsKey(streamId)).collect(Collectors.toList()).isEmpty());
    Set<String> intermediateStreams = new HashSet<>(inputDescriptors.keySet());
    jobGraph.getInputStreams().forEach(edge -> {
        if (intermediateStreams.contains(edge.getStreamSpec().getId())) {
            intermediateStreams.remove(edge.getStreamSpec().getId());
        }
    });
    assertEquals(new HashSet<>(Arrays.asList(intermediateStream1, intermediateBroadcast)), intermediateStreams);
}