Examples with StreamlineEvent com.hortonworks.streamline.streams.StreamlineEvent used on opensource projects

Example 6 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class StreamlineTimestampExtractor method extractTimestamp.

@Override
public long extractTimestamp(Tuple tuple) {
    StreamlineEvent event = (StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    Object ts = event.get(fieldName);
    if (ts == null || !(ts instanceof Long)) {
        throw new IllegalArgumentException("Streamline event does not contain a long value in field: " + fieldName);
    }
    return (long) ts;
}

Example 7 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class NormalizationBolt method process.

public void process(Tuple inputTuple, StreamlineEvent event) throws Exception {
    LOG.debug("Normalizing received StreamlineEvent: [{}] with tuple: [{}]", event, inputTuple);
    // todo this bolt will be replaced with custom baseprocessor bolt.
    StreamlineEventImpl eventWithStream = StreamlineEventImpl.builder().from(event).sourceStream(inputTuple.getSourceStreamId()).build();
    List<Result> outputEvents = normalizationProcessorRuntime.process(eventWithStream);
    LOG.debug("Emitting events to collector: [{}]", outputEvents);
    for (Result outputEvent : outputEvents) {
        for (StreamlineEvent e : outputEvent.events) {
            collector.emit(outputEvent.stream, inputTuple, new Values(e));
        }
    }
}

Example 8 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class CustomProcessorBolt method process.

@Override
protected void process(Tuple input, StreamlineEvent event) {
    try {
        StreamlineEvent toProcess = null;
        if (inputSchemaMap == null || inputSchemaMap.isEmpty() || !inputSchemaMap.containsKey(input.getSourceStreamId())) {
            toProcess = event;
        } else {
            // Create a new mapped event based on mapping to pass it to CP implementation
            Map<String, Object> mappedEventMap = new HashMap<>();
            for (Map.Entry<String, String> entry : inputSchemaMap.get(input.getSourceStreamId()).entrySet()) {
                if (event.get(entry.getValue()) != null) {
                    mappedEventMap.put(entry.getKey(), event.get(entry.getValue()));
                }
            }
            toProcess = StreamlineEventImpl.builder().from(event).sourceStream(input.getSourceStreamId()).fieldsAndValues(mappedEventMap).build();
        }
        List<StreamlineEvent> results = customProcessorRuntime.process(toProcess);
        if (results != null) {
            Schema schema = outputSchema.values().iterator().next();
            String outputStream = outputSchema.keySet().iterator().next();
            for (StreamlineEvent e : results) {
                Map<String, Object> newFieldsAndValues = new HashMap<>();
                // event and CP defined output schema. UI will make sure that the fields are from one of the two sets.
                for (Schema.Field field : schema.getFields()) {
                    // value has to be present either in the input event
                    newFieldsAndValues.put(field.getName(), e.containsKey(field.getName()) ? e.get(field.getName()) : event.get(field.getName()));
                }
                StreamlineEvent toEmit = StreamlineEventImpl.builder().from(e).sourceStream(outputStream).fieldsAndValues(newFieldsAndValues).build();
                collector.emit(outputStream, input, new Values(toEmit));
            }
        }
    } catch (ProcessingException e) {
        LOG.error("Custom Processor threw a ProcessingException. ", e);
        throw new RuntimeException(e);
    }
}

Example 9 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class StreamlineHiveMapper method mapPartitions.

@Override
public List<String> mapPartitions(Tuple tuple) {
    StreamlineEvent event = (StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    List<String> partitionList = new ArrayList<>();
    for (String field : this.partitionFields) {
        Object val = event.get(field);
        if (val == null) {
            throw new IllegalArgumentException(String.format("Partition field '%s' value is missing in the streamline event", field));
        }
        partitionList.add(val.toString());
    }
    if (this.timeFormat != null) {
        partitionList.add(parseDate.format(System.currentTimeMillis()));
    }
    return partitionList;
}

Example 10 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class StreamlineHiveMapper method mapRecord.

@Override
public byte[] mapRecord(Tuple tuple) {
    StreamlineEvent event = (StreamlineEvent) tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
    StringBuilder builder = new StringBuilder();
    for (String field : this.fields) {
        Object val = event.get(field);
        if (val == null) {
            throw new IllegalArgumentException(String.format("Field '%s' value is missing in the streamline event", field));
        }
        builder.append(val);
        builder.append(fieldDelimiter);
    }
    return builder.toString().getBytes();
}