Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 56 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class StreamAppender method setupSystem.

/**
 * This should only be called after verifying that the {@link LoggingContextHolder} has the config.
 */
protected void setupSystem() {
    config = getConfig();
    Log4jSystemConfig log4jSystemConfig = getLog4jSystemConfig(config);
    if (streamName == null) {
        streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
    }
    // Instantiate metrics
    MetricsRegistryMap metricsRegistry = new MetricsRegistryMap();
    // Take this.getClass().getName() as the name to make it extend-friendly
    metrics = getMetrics(metricsRegistry);
    // Register metrics into metrics reporters so that they are able to be reported to other systems
    Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), containerName);
    metricsReporters.values().forEach(reporter -> {
        reporter.register(containerName, metricsRegistry);
        reporter.start();
    });
    String systemName = log4jSystemConfig.getSystemName();
    String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName).orElseThrow(() -> new SamzaException("Could not figure out \"" + systemName + "\" system factory for log4j " + getName() + " to use"));
    SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);
    setSerde(log4jSystemConfig, systemName);
    setupStream(systemFactory, systemName);
    systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry, this.getClass().getSimpleName());
    systemStream = new SystemStream(systemName, streamName);
    systemProducer.register(SOURCE);
    systemProducer.start();
    System.out.println(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName + " in " + systemName + ". Logs are partitioned by " + key);
    startTransferThread();
}

Example 57 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestStartpoint method createCoordinatorStreamStore.

private static CoordinatorStreamStore createCoordinatorStreamStore(Config applicationConfig) {
    SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(applicationConfig);
    SystemAdmins systemAdmins = new SystemAdmins(applicationConfig);
    SystemAdmin coordinatorSystemAdmin = systemAdmins.getSystemAdmin(coordinatorSystemStream.getSystem());
    coordinatorSystemAdmin.start();
    CoordinatorStreamUtil.createCoordinatorStream(coordinatorSystemStream, coordinatorSystemAdmin);
    coordinatorSystemAdmin.stop();
    return new CoordinatorStreamStore(applicationConfig, new NoOpMetricsRegistry());
}

Example 58 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class RestCall method run.

@Override
public void run() {
    try {
        // Let the thread sleep for a while.
        Thread.sleep(random.nextInt(150));
    } catch (InterruptedException e) {
        System.out.println("Thread " + this.getName() + " interrupted.");
    }
    Integer obj = (Integer) envelope.getMessage();
    messageCollector.send(new OutgoingMessageEnvelope(new SystemStream("async-test", "ints-out"), envelope.getKey(), envelope.getKey(), obj * 10));
    callback.complete();
}

Example 59 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class Emitter method window.

public void window(MessageCollector collector, TaskCoordinator coordinator) {
    Integer epoch = getInt(EPOCH);
    if (epoch == null) {
        resetEpoch();
        return;
    }
    int counter = getInt(COUNT);
    if (counter < max) {
        logger.info("Emitting: " + counter + ", epoch = " + epoch + ", task = " + taskName);
        OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(new SystemStream("kafka", "emitted"), Integer.toString(counter), epoch + "-" + taskName.toString());
        collector.send(envelope);
        this.state.put(COUNT, Integer.toString(getInt(COUNT) + 1));
    }
}

Example 60 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class RepartitionJoinWindowApp method describe.

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
    // offset.default = oldest required for tests since checkpoint topic is empty on start and messages are published
    // before the application is run
    Config config = appDescriptor.getConfig();
    String inputTopic1 = config.get(INPUT_TOPIC_1_CONFIG_KEY);
    String inputTopic2 = config.get(INPUT_TOPIC_2_CONFIG_KEY);
    String outputTopic = config.get(OUTPUT_TOPIC_CONFIG_KEY);
    KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
    KafkaInputDescriptor<PageView> id1 = ksd.getInputDescriptor(inputTopic1, new JsonSerdeV2<>(PageView.class));
    KafkaInputDescriptor<AdClick> id2 = ksd.getInputDescriptor(inputTopic2, new JsonSerdeV2<>(AdClick.class));
    MessageStream<PageView> pageViews = appDescriptor.getInputStream(id1);
    MessageStream<AdClick> adClicks = appDescriptor.getInputStream(id2);
    MessageStream<KV<String, PageView>> pageViewsRepartitionedByViewId = pageViews.partitionBy(PageView::getViewId, pv -> pv, new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(PageView.class)), "pageViewsByViewId");
    MessageStream<PageView> pageViewsRepartitionedByViewIdValueONly = pageViewsRepartitionedByViewId.map(KV::getValue);
    MessageStream<KV<String, AdClick>> adClicksRepartitionedByViewId = adClicks.partitionBy(AdClick::getViewId, ac -> ac, new KVSerde<>(new StringSerde(), new JsonSerdeV2<>(AdClick.class)), "adClicksByViewId");
    MessageStream<AdClick> adClicksRepartitionedByViewIdValueOnly = adClicksRepartitionedByViewId.map(KV::getValue);
    MessageStream<UserPageAdClick> userPageAdClicks = pageViewsRepartitionedByViewIdValueONly.join(adClicksRepartitionedByViewIdValueOnly, new UserPageViewAdClicksJoiner(), new StringSerde(), new JsonSerdeV2<>(PageView.class), new JsonSerdeV2<>(AdClick.class), Duration.ofMinutes(1), "pageViewAdClickJoin");
    MessageStream<KV<String, UserPageAdClick>> userPageAdClicksByUserId = userPageAdClicks.partitionBy(UserPageAdClick::getUserId, upac -> upac, KVSerde.of(new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userPageAdClicksByUserId");
    userPageAdClicksByUserId.map(KV::getValue).window(Windows.keyedSessionWindow(UserPageAdClick::getUserId, Duration.ofSeconds(3), new StringSerde(), new JsonSerdeV2<>(UserPageAdClick.class)), "userAdClickWindow").map(windowPane -> KV.of(windowPane.getKey().getKey(), String.valueOf(windowPane.getMessage().size()))).sink((message, messageCollector, taskCoordinator) -> {
        taskCoordinator.commit(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
        messageCollector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", outputTopic), null, message.getKey(), message.getValue()));
    });
    intermediateStreamIds.add(((IntermediateMessageStreamImpl) pageViewsRepartitionedByViewId).getStreamId());
    intermediateStreamIds.add(((IntermediateMessageStreamImpl) adClicksRepartitionedByViewId).getStreamId());
    intermediateStreamIds.add(((IntermediateMessageStreamImpl) userPageAdClicksByUserId).getStreamId());
}