Search in sources :

Example 1 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class NormalizationBoltTest method createBulkNormalizationProcessor.

public static NormalizationProcessor createBulkNormalizationProcessor(String outputStreamId) throws NormalizationException, IOException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema inputSchema = Schema.of(Schema.Field.of("temp", Schema.Type.INTEGER), Schema.Field.of("foo", Schema.Type.STRING));
    String bulkScriptText = getBulkScriptText();
    BulkNormalizationConfig bulkNormalizationConfig = new BulkNormalizationConfig(inputSchema, bulkScriptText);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, bulkNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.bulk);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}
Also used : BulkNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.BulkNormalizationConfig) NormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig) FieldBasedNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldBasedNormalizationConfig) BulkNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.BulkNormalizationConfig) HashMap(java.util.HashMap) Schema(com.hortonworks.registries.common.Schema) NormalizationProcessor(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationProcessor) Stream(com.hortonworks.streamline.streams.layout.component.Stream) InputStream(java.io.InputStream)

Example 2 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class TopologyComponentFactory method normalizationProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> normalizationProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Config config = component.getConfig();
            Object typeObj = config.getAny(NormalizationProcessor.CONFIG_KEY_TYPE);
            Object normConfObj = config.getAny(NormalizationProcessor.CONFIG_KEY_NORMALIZATION);
            ObjectMapper objectMapper = new ObjectMapper();
            NormalizationProcessor.Type type = objectMapper.convertValue(typeObj, NormalizationProcessor.Type.class);
            Map<String, NormalizationConfig> normConfig = objectMapper.convertValue(normConfObj, new TypeReference<Map<String, NormalizationConfig>>() {
            });
            updateWithSchemas(component.getTopologyId(), component.getVersionId(), normConfig);
            Set<Stream> outputStreams;
            if (component instanceof TopologyOutputComponent) {
                outputStreams = createOutputStreams((TopologyOutputComponent) component);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            if (outputStreams.size() != 1) {
                throw new IllegalArgumentException("Normalization component [" + component + "] must have only one output stream");
            }
            return new NormalizationProcessor(normConfig, outputStreams.iterator().next(), type);
        }
    };
    return new SimpleImmutableEntry<>(NORMALIZATION, provider);
}
Also used : TopologyComponent(com.hortonworks.streamline.streams.catalog.TopologyComponent) NormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig) Config(com.hortonworks.streamline.common.Config) NormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig) NormalizationProcessor(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationProcessor) StreamlineProcessor(com.hortonworks.streamline.streams.layout.component.StreamlineProcessor) TopologyOutputComponent(com.hortonworks.streamline.streams.catalog.TopologyOutputComponent) SimpleImmutableEntry(java.util.AbstractMap.SimpleImmutableEntry) TopologyStream(com.hortonworks.streamline.streams.catalog.TopologyStream) Stream(com.hortonworks.streamline.streams.layout.component.Stream) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper)

Example 3 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class NormalizationBoltTest method createFieldBasedNormalizationProcessor.

public static NormalizationProcessor createFieldBasedNormalizationProcessor(String outputStreamId) throws NormalizationException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema.Field tempField = new Schema.Field("temp", Schema.Type.INTEGER);
    Schema inputSchema = Schema.of(tempField, new Schema.Field("foo", Schema.Type.STRING));
    Transformer transformer = new Transformer(tempField, new Schema.Field("temperature", Schema.Type.FLOAT));
    transformer.setConverterScript("new Float((temp-32)*5/9f)");
    List<Transformer> transformers = Collections.singletonList(transformer);
    List<String> filters = Collections.singletonList("foo");
    List<FieldValueGenerator> fieldValueGenerators = Collections.singletonList(new FieldValueGenerator(new Schema.Field("new-field", Schema.Type.STRING), "new value"));
    FieldBasedNormalizationConfig fieldBasedNormalizationConfig = new FieldBasedNormalizationConfig(inputSchema, transformers, filters, fieldValueGenerators);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, fieldBasedNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.fineGrained);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}
Also used : Transformer(com.hortonworks.streamline.streams.layout.component.impl.normalization.Transformer) NormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig) FieldBasedNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldBasedNormalizationConfig) BulkNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.BulkNormalizationConfig) HashMap(java.util.HashMap) Schema(com.hortonworks.registries.common.Schema) NormalizationProcessor(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationProcessor) FieldValueGenerator(com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldValueGenerator) FieldBasedNormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldBasedNormalizationConfig) Stream(com.hortonworks.streamline.streams.layout.component.Stream) InputStream(java.io.InputStream)

Example 4 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class TopologyComponentFactory method updateWithSchemas.

private void updateWithSchemas(Long topologyId, Long versionId, Map<String, NormalizationConfig> normalizationConfigRead) {
    for (Map.Entry<String, NormalizationConfig> entry : normalizationConfigRead.entrySet()) {
        NormalizationConfig normalizationConfig = entry.getValue();
        TopologyStream topologyStream = catalogService.getStreamInfoByName(topologyId, entry.getKey(), versionId);
        if (topologyStream != null) {
            normalizationConfig.setInputSchema(topologyStream.getSchema());
        }
    }
}
Also used : NormalizationConfig(com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig) TopologyStream(com.hortonworks.streamline.streams.catalog.TopologyStream) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap)

Aggregations

NormalizationConfig (com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig)4 Stream (com.hortonworks.streamline.streams.layout.component.Stream)3 NormalizationProcessor (com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationProcessor)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 Schema (com.hortonworks.registries.common.Schema)2 TopologyStream (com.hortonworks.streamline.streams.catalog.TopologyStream)2 BulkNormalizationConfig (com.hortonworks.streamline.streams.layout.component.impl.normalization.BulkNormalizationConfig)2 FieldBasedNormalizationConfig (com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldBasedNormalizationConfig)2 InputStream (java.io.InputStream)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 Config (com.hortonworks.streamline.common.Config)1 TopologyComponent (com.hortonworks.streamline.streams.catalog.TopologyComponent)1 TopologyOutputComponent (com.hortonworks.streamline.streams.catalog.TopologyOutputComponent)1 StreamlineProcessor (com.hortonworks.streamline.streams.layout.component.StreamlineProcessor)1 FieldValueGenerator (com.hortonworks.streamline.streams.layout.component.impl.normalization.FieldValueGenerator)1 Transformer (com.hortonworks.streamline.streams.layout.component.impl.normalization.Transformer)1 SimpleImmutableEntry (java.util.AbstractMap.SimpleImmutableEntry)1