Examples with StreamlineEvent com.hortonworks.streamline.streams.StreamlineEvent used on opensource projects

Example 26 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class SubstituteTransformRuntimeTest method testSubstituteSimpleVars.

@Test
public void testSubstituteSimpleVars() throws Exception {
    Map<String, Object> fieldsAndValues = new HashMap<>();
    fieldsAndValues.put("1", "one");
    fieldsAndValues.put("2", "two");
    fieldsAndValues.put("3", "${1} plus ${2}");
    StreamlineEvent event = StreamlineEventImpl.builder().fieldsAndValues(fieldsAndValues).dataSourceId("dsrcid").build();
    TransformRuntime transformRuntime = new SubstituteTransformRuntime();
    List<StreamlineEvent> result = transformRuntime.execute(event);
    assertEquals(1, result.size());
    assertEquals(3, result.get(0).size());
    assertEquals("one", result.get(0).get("1"));
    assertEquals("two", result.get(0).get("2"));
    assertEquals("one plus two", result.get(0).get("3"));
}

Example 27 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class NormalizationProcessorRuntime method process.

/*
     * todo: It should receive input Stream also and generate output Stream along with StreamlineEvent. This support should
     * come from processor framework, will add later.
     */
@Override
public List<Result> process(StreamlineEvent event) throws ProcessingException {
    String currentStreamId = event.getSourceStream() != null ? event.getSourceStream() : NormalizationProcessor.DEFAULT_STREAM_ID;
    NormalizationRuntime normalizationRuntime = schemasWithNormalizationRuntime.get(currentStreamId);
    LOG.debug("Normalization runtime for this stream [{}]", normalizationRuntime);
    StreamlineEvent outputEvent = event;
    if (normalizationRuntime != null) {
        try {
            outputEvent = normalizationRuntime.execute(event);
            schemaValidator.validate(outputEvent);
        } catch (NormalizationException e) {
            throw new RuntimeException(e);
        }
    } else {
        LOG.debug("No normalization defined for stream: [{}]", currentStreamId);
    }
    // which does not have normalization configuration.
    return Collections.singletonList(new Result(NormalizationProcessor.DEFAULT_STREAM_ID, Collections.singletonList(outputEvent)));
}

Example 28 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class DefaultJoiner method join.

@Override
public StreamlineEvent join(EventGroup eventGroup) {
    Map<String, Object> fieldValues = new HashMap<>();
    Map<String, Object> auxiliaryFieldValues = new HashMap<>();
    for (StreamlineEvent subEvent : eventGroup.getSplitEvents()) {
        if (subEvent.getAuxiliaryFieldsAndValues() != null) {
            auxiliaryFieldValues.putAll(subEvent.getAuxiliaryFieldsAndValues());
        }
        fieldValues.putAll(subEvent);
    }
    return StreamlineEventImpl.builder().fieldsAndValues(fieldValues).dataSourceId(eventGroup.getDataSourceId()).auxiliaryFieldsAndValues(auxiliaryFieldValues).build();
}

Example 29 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class JoinActionRuntime method joinEvents.

/**
 * Join all subevents and generate an event for the given output stream.
 *
 * @param eventGroup
 */
protected List<Result> joinEvents(EventGroup eventGroup) {
    StreamlineEvent joinedEvent = joiner.join(eventGroup);
    List<Result> results = new ArrayList<>();
    for (String stream : getOutputStreams()) {
        results.add(new Result(stream, Collections.singletonList(getStreamlineEvent(joinedEvent, stream))));
    }
    groupedEvents.invalidate(eventGroup.getGroupId());
    return results;
}

Example 30 with StreamlineEvent

use of com.hortonworks.streamline.streams.StreamlineEvent in project streamline by hortonworks.

the class CustomProcessorBoltTest method testExecute.

private void testExecute(boolean isSuccess) throws ProcessingException, ClassNotFoundException, MalformedURLException, InstantiationException, IllegalAccessException {
    customProcessorBolt.customProcessorImpl(ConsoleCustomProcessor.class.getCanonicalName());
    customProcessorBolt.outputSchema(outputStreamToSchema);
    Map<String, Object> data = new HashMap<>();
    data.put("key", "value");
    final StreamlineEvent event = StreamlineEventImpl.builder().fieldsAndValues(data).dataSourceId("dsrcid").build();
    final List<StreamlineEvent> result = Arrays.asList(event);
    final ProcessingException pe = new ProcessingException("Test");
    new Expectations() {

        {
            tuple.getSourceComponent();
            returns("datasource");
            tuple.getSourceStreamId();
            returns(stream);
            tuple.getValueByField(StreamlineEvent.STREAMLINE_EVENT);
            returns(event);
        }
    };
    if (!isSuccess) {
        new Expectations() {

            {
                customProcessorRuntime.process(event);
                result = pe;
            }
        };
    } else {
        new Expectations() {

            {
                customProcessorRuntime.process(event);
                returns(result);
            }
        };
    }
    Map<Object, Object> conf = new HashMap<>();
    customProcessorBolt.prepare(conf, null, mockOutputCollector);
    customProcessorBolt.execute(tuple);
    if (!isSuccess) {
        new VerificationsInOrder() {

            {
                RuntimeException e;
                customProcessorRuntime.process(event);
                times = 1;
                mockOutputCollector.fail(tuple);
                times = 1;
                mockOutputCollector.reportError(e = withCapture());
                assertTrue(e.getCause() == pe);
            }
        };
    } else {
        new VerificationsInOrder() {

            {
                tuple.getSourceComponent();
                times = 1;
                tuple.getSourceStreamId();
                times = 1;
                StreamlineEvent actual;
                customProcessorRuntime.process(actual = withCapture());
                times = 1;
                Assert.assertEquals(actual.getSourceStream(), stream);
                Assert.assertEquals(actual, event);
                Values actualValues;
                mockOutputCollector.emit(outputStream, tuple, actualValues = withCapture());
                times = 1;
                mockOutputCollector.ack(tuple);
                times = 1;
            }
        };
    }
}