Examples with NormalizationConfig com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig used on opensource projects

Example 1 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class NormalizationBoltTest method createBulkNormalizationProcessor.

public static NormalizationProcessor createBulkNormalizationProcessor(String outputStreamId) throws NormalizationException, IOException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema inputSchema = Schema.of(Schema.Field.of("temp", Schema.Type.INTEGER), Schema.Field.of("foo", Schema.Type.STRING));
    String bulkScriptText = getBulkScriptText();
    BulkNormalizationConfig bulkNormalizationConfig = new BulkNormalizationConfig(inputSchema, bulkScriptText);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, bulkNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.bulk);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}

Example 2 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class TopologyComponentFactory method normalizationProcessorProvider.

private Map.Entry<String, Provider<StreamlineProcessor>> normalizationProcessorProvider() {
    Provider<StreamlineProcessor> provider = new Provider<StreamlineProcessor>() {

        @Override
        public StreamlineProcessor create(TopologyComponent component) {
            Config config = component.getConfig();
            Object typeObj = config.getAny(NormalizationProcessor.CONFIG_KEY_TYPE);
            Object normConfObj = config.getAny(NormalizationProcessor.CONFIG_KEY_NORMALIZATION);
            ObjectMapper objectMapper = new ObjectMapper();
            NormalizationProcessor.Type type = objectMapper.convertValue(typeObj, NormalizationProcessor.Type.class);
            Map<String, NormalizationConfig> normConfig = objectMapper.convertValue(normConfObj, new TypeReference<Map<String, NormalizationConfig>>() {
            });
            updateWithSchemas(component.getTopologyId(), component.getVersionId(), normConfig);
            Set<Stream> outputStreams;
            if (component instanceof TopologyOutputComponent) {
                outputStreams = createOutputStreams((TopologyOutputComponent) component);
            } else {
                throw new IllegalArgumentException("Component " + component + " must be an instance of TopologyOutputComponent");
            }
            if (outputStreams.size() != 1) {
                throw new IllegalArgumentException("Normalization component [" + component + "] must have only one output stream");
            }
            return new NormalizationProcessor(normConfig, outputStreams.iterator().next(), type);
        }
    };
    return new SimpleImmutableEntry<>(NORMALIZATION, provider);
}

Example 3 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class NormalizationBoltTest method createFieldBasedNormalizationProcessor.

public static NormalizationProcessor createFieldBasedNormalizationProcessor(String outputStreamId) throws NormalizationException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema.Field tempField = new Schema.Field("temp", Schema.Type.INTEGER);
    Schema inputSchema = Schema.of(tempField, new Schema.Field("foo", Schema.Type.STRING));
    Transformer transformer = new Transformer(tempField, new Schema.Field("temperature", Schema.Type.FLOAT));
    transformer.setConverterScript("new Float((temp-32)*5/9f)");
    List<Transformer> transformers = Collections.singletonList(transformer);
    List<String> filters = Collections.singletonList("foo");
    List<FieldValueGenerator> fieldValueGenerators = Collections.singletonList(new FieldValueGenerator(new Schema.Field("new-field", Schema.Type.STRING), "new value"));
    FieldBasedNormalizationConfig fieldBasedNormalizationConfig = new FieldBasedNormalizationConfig(inputSchema, transformers, filters, fieldValueGenerators);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, fieldBasedNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.fineGrained);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}

Example 4 with NormalizationConfig

use of com.hortonworks.streamline.streams.layout.component.impl.normalization.NormalizationConfig in project streamline by hortonworks.

the class TopologyComponentFactory method updateWithSchemas.

private void updateWithSchemas(Long topologyId, Long versionId, Map<String, NormalizationConfig> normalizationConfigRead) {
    for (Map.Entry<String, NormalizationConfig> entry : normalizationConfigRead.entrySet()) {
        NormalizationConfig normalizationConfig = entry.getValue();
        TopologyStream topologyStream = catalogService.getStreamInfoByName(topologyId, entry.getKey(), versionId);
        if (topologyStream != null) {
            normalizationConfig.setInputSchema(topologyStream.getSchema());
        }
    }
}