Examples with SystemProducer org.apache.samza.system.SystemProducer used on opensource projects

Example 6 with SystemProducer

use of org.apache.samza.system.SystemProducer in project beam by apache.

the class TranslationContext method createDummyStreamDescriptor.

/**
 * The dummy stream created will only be used in Beam tests.
 */
private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) {
    final GenericSystemDescriptor dummySystem = new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName());
    final GenericInputDescriptor<OpMessage<String>> dummyInput = dummySystem.getInputDescriptor(id, new NoOpSerde<>());
    dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST);
    final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig());
    final SystemFactory factory = new InMemorySystemFactory();
    final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1);
    factory.getAdmin(id, config).createStream(dummyStreamSpec);
    final SystemProducer producer = factory.getProducer(id, config, null);
    final SystemStream sysStream = new SystemStream(id, id);
    final Consumer<Object> sendFn = (msg) -> {
        producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg));
    };
    final WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant());
    sendFn.accept(OpMessage.ofElement(windowedValue));
    sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
    sendFn.accept(new EndOfStreamMessage(null));
    return dummyInput;
}

Example 7 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class AsyncSystemProducer method flush.

/**
 * Default implementation of the flush that just waits for all the pendingFutures to be complete.
 * SystemProducer should override this, if the underlying system provides flush semantics.
 * @param source String representing the source of the message.
 */
@Override
public synchronized void flush(String source) {
    long incompleteSends = pendingFutures.stream().filter(x -> !x.isDone()).count();
    LOG.info("Trying to flush pending {} sends.", incompleteSends);
    checkForSendCallbackErrors("Received exception on message send.");
    CompletableFuture<Void> future = CompletableFuture.allOf(pendingFutures.toArray(new CompletableFuture[pendingFutures.size()]));
    try {
        // Block until all the pending sends are complete or timeout.
        future.get(DEFAULT_FLUSH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
        incompleteSends = pendingFutures.stream().filter(x -> !x.isDone()).count();
        String msg = String.format("Flush failed with error. Total pending sends %d", incompleteSends);
        LOG.error(msg, e);
        throw new SamzaException(msg, e);
    }
    pendingFutures.clear();
    checkForSendCallbackErrors("Sending one or more of the messages failed during flush.");
}

Example 8 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class SystemProducerBench method start.

public void start() throws IOException, InterruptedException {
    super.start();
    String source = "SystemProducerBench";
    int size = Integer.parseInt(cmd.getOptionValue(OPT_SHORT_MESSAGE_SIZE));
    RandomValueGenerator randGenerator = new RandomValueGenerator(System.currentTimeMillis());
    value = randGenerator.getNextString(size, size).getBytes();
    NoOpMetricsRegistry metricsRegistry = new NoOpMetricsRegistry();
    List<SystemStreamPartition> ssps = createSSPs(systemName, physicalStreamName, startPartition, endPartition);
    SystemProducer producer = factory.getProducer(systemName, config, metricsRegistry);
    producer.register(source);
    producer.start();
    System.out.println("starting production at " + Instant.now());
    Instant startTime = Instant.now();
    for (int index = 0; index < totalEvents; index++) {
        SystemStreamPartition ssp = ssps.get(index % ssps.size());
        OutgoingMessageEnvelope messageEnvelope = createMessageEnvelope(ssp, index);
        producer.send(source, messageEnvelope);
    }
    System.out.println("Ending production at " + Instant.now());
    System.out.println(String.format("Event Rate is %s Messages/Sec", totalEvents * 1000 / Duration.between(startTime, Instant.now()).toMillis()));
    producer.flush(source);
    System.out.println("Ending flush at " + Instant.now());
    System.out.println(String.format("Event Rate with flush is %s Messages/Sec", totalEvents * 1000 / Duration.between(startTime, Instant.now()).toMillis()));
    producer.stop();
    System.exit(0);
}

Example 9 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestInMemorySystem method testNullMessageWithValidMessageKey.

@Test
public void testNullMessageWithValidMessageKey() {
    final String messageKey = "validKey";
    SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry);
    systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, messageKey, null));
    SystemConsumer consumer = systemFactory.getConsumer(SYSTEM_NAME, config, mockRegistry);
    Set<SystemStreamPartition> sspsToPoll = IntStream.range(0, PARTITION_COUNT).mapToObj(partition -> new SystemStreamPartition(SYSTEM_STREAM, new Partition(partition))).collect(Collectors.toSet());
    // register the consumer for ssps
    for (SystemStreamPartition ssp : sspsToPoll) {
        consumer.register(ssp, "0");
    }
    List<IncomingMessageEnvelope> results = consumeRawMessages(consumer, sspsToPoll);
    assertEquals(1, results.size());
    assertEquals(results.get(0).getKey(), messageKey);
    assertNull(results.get(0).getMessage());
}

Example 10 with SystemProducer

use of org.apache.samza.system.SystemProducer in project samza by apache.

the class TestInMemorySystem method testNullMessageWithNullKey.

@Test(expected = NullPointerException.class)
public void testNullMessageWithNullKey() {
    SystemProducer systemProducer = systemFactory.getProducer(SYSTEM_NAME, config, mockRegistry);
    systemProducer.send(SOURCE, new OutgoingMessageEnvelope(SYSTEM_STREAM, null));
}