Examples with StreamConfig org.apache.samza.config.StreamConfig used on opensource projects

Example 36 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class OperatorImplGraph method createOperatorImpl.

/**
 * Creates a new {@link OperatorImpl} instance for the provided {@link OperatorSpec}.
 *
 * @param prevOperatorSpec the original {@link OperatorSpec} that produces output for {@code operatorSpec} from {@link OperatorSpecGraph}
 * @param operatorSpec  the original {@link OperatorSpec} from {@link OperatorSpecGraph}
 * @param context  the {@link Context} required to instantiate operators
 * @return  the {@link OperatorImpl} implementation instance
 */
OperatorImpl createOperatorImpl(OperatorSpec prevOperatorSpec, OperatorSpec operatorSpec, Context context) {
    Config config = context.getJobContext().getConfig();
    StreamConfig streamConfig = new StreamConfig(config);
    if (operatorSpec instanceof InputOperatorSpec) {
        return new InputOperatorImpl((InputOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof StreamOperatorSpec) {
        return new FlatmapOperatorImpl((StreamOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof SinkOperatorSpec) {
        return new SinkOperatorImpl((SinkOperatorSpec) operatorSpec);
    } else if (operatorSpec instanceof OutputOperatorSpec) {
        String streamId = ((OutputOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new OutputOperatorImpl((OutputOperatorSpec) operatorSpec, systemStream);
    } else if (operatorSpec instanceof PartitionByOperatorSpec) {
        String streamId = ((PartitionByOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new PartitionByOperatorImpl((PartitionByOperatorSpec) operatorSpec, systemStream, internalTaskContext);
    } else if (operatorSpec instanceof WindowOperatorSpec) {
        return new WindowOperatorImpl((WindowOperatorSpec) operatorSpec, clock);
    } else if (operatorSpec instanceof JoinOperatorSpec) {
        return getOrCreatePartialJoinOpImpls((JoinOperatorSpec) operatorSpec, prevOperatorSpec.equals(((JoinOperatorSpec) operatorSpec).getLeftInputOpSpec()), clock);
    } else if (operatorSpec instanceof StreamTableJoinOperatorSpec) {
        return new StreamTableJoinOperatorImpl((StreamTableJoinOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof SendToTableOperatorSpec) {
        return new SendToTableOperatorImpl((SendToTableOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof SendToTableWithUpdateOperatorSpec) {
        return new SendToTableWithUpdateOperatorImpl((SendToTableWithUpdateOperatorSpec) operatorSpec, context);
    } else if (operatorSpec instanceof BroadcastOperatorSpec) {
        String streamId = ((BroadcastOperatorSpec) operatorSpec).getOutputStream().getStreamId();
        SystemStream systemStream = streamConfig.streamIdToSystemStream(streamId);
        return new BroadcastOperatorImpl((BroadcastOperatorSpec) operatorSpec, systemStream, context);
    } else if (operatorSpec instanceof AsyncFlatMapOperatorSpec) {
        return new AsyncFlatmapOperatorImpl((AsyncFlatMapOperatorSpec) operatorSpec);
    }
    throw new IllegalArgumentException(String.format("Unsupported OperatorSpec: %s", operatorSpec.getClass().getName()));
}

Example 37 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class MetricsSnapshotReporterFactory method getSerde.

protected Serde<MetricsSnapshot> getSerde(String reporterName, Config config) {
    StreamConfig streamConfig = new StreamConfig(config);
    SystemConfig systemConfig = new SystemConfig(config);
    SystemStream systemStream = getSystemStream(reporterName, config);
    Optional<String> streamSerdeName = streamConfig.getStreamMsgSerde(systemStream);
    Optional<String> systemSerdeName = systemConfig.getSystemMsgSerde(systemStream.getSystem());
    String serdeName = streamSerdeName.orElse(systemSerdeName.orElse(null));
    SerializerConfig serializerConfig = new SerializerConfig(config);
    Serde<MetricsSnapshot> serde;
    if (serdeName != null) {
        Optional<String> serdeFactoryClass = serializerConfig.getSerdeFactoryClass(serdeName);
        if (serdeFactoryClass.isPresent()) {
            SerdeFactory<MetricsSnapshot> serdeFactory = ReflectionUtil.getObj(serdeFactoryClass.get(), SerdeFactory.class);
            serde = serdeFactory.getSerde(serdeName, config);
        } else {
            serde = null;
        }
    } else {
        serde = new MetricsSnapshotSerdeV2();
    }
    LOG.info("Got serde {}.", serde);
    return serde;
}

Example 38 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class JobPlanner method getGeneratedConfig.

private Map<String, String> getGeneratedConfig() {
    Map<String, String> generatedConfig = new HashMap<>();
    Map<String, String> systemStreamConfigs = generateSystemStreamConfigs(appDesc);
    generatedConfig.putAll(systemStreamConfigs);
    StreamConfig streamConfig = new StreamConfig(new MapConfig(generatedConfig));
    Set<String> inputStreamIds = new HashSet<>(appDesc.getInputStreamIds());
    // exclude intermediate streams
    inputStreamIds.removeAll(appDesc.getOutputStreamIds());
    final ApplicationConfig.ApplicationMode mode;
    if (inputStreamIds.isEmpty()) {
        // use stream by default
        mode = ApplicationConfig.ApplicationMode.STREAM;
    } else {
        mode = inputStreamIds.stream().allMatch(streamConfig::getIsBounded) ? ApplicationConfig.ApplicationMode.BATCH : ApplicationConfig.ApplicationMode.STREAM;
    }
    generatedConfig.put(ApplicationConfig.APP_MODE, mode.name());
    // adding app.class in the configuration, unless it is LegacyTaskApplication
    if (!LegacyTaskApplication.class.getName().equals(appDesc.getAppClass().getName())) {
        generatedConfig.put(ApplicationConfig.APP_CLASS, appDesc.getAppClass().getName());
    }
    return generatedConfig;
}

Example 39 with StreamConfig

use of org.apache.samza.config.StreamConfig in project samza by apache.

the class KafkaSystemAdmin method getIntermediateStreamProperties.

/**
 * Fetch stream properties for all intermediate streams.
 *
 * @param config kafka system config
 * @return a {@link Map} from {@code streamId} to stream {@link Properties}
 */
@VisibleForTesting
static Map<String, Properties> getIntermediateStreamProperties(Config config) {
    Map<String, Properties> intermedidateStreamProperties = Collections.emptyMap();
    ApplicationConfig appConfig = new ApplicationConfig(config);
    if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) {
        StreamConfig streamConfig = new StreamConfig(config);
        intermedidateStreamProperties = streamConfig.getStreamIds().stream().filter(streamConfig::getIsIntermediateStream).collect(Collectors.toMap(Function.identity(), streamId -> {
            Properties properties = new Properties();
            properties.putAll(streamConfig.getStreamProperties(streamId));
            properties.putIfAbsent(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH()));
            return properties;
        }));
    }
    return intermedidateStreamProperties;
}