Examples with Stream com.hortonworks.streamline.streams.layout.component.Stream used on opensource projects

Example 11 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class StormTopologyFluxGenerator method visit.

@Override
public void visit(final RulesProcessor rulesProcessor) {
    rulesProcessor.getConfig().setAny("outputStreams", rulesProcessor.getOutputStreams());
    List<Rule> rulesWithWindow = new ArrayList<>();
    List<Rule> rulesWithoutWindow = new ArrayList<>();
    Set<String> inStreams = topologyDag.getEdgesTo(rulesProcessor).stream().flatMap(e -> e.getStreamGroupings().stream().map(sg -> sg.getStream().getId())).collect(Collectors.toSet());
    for (Rule rule : rulesProcessor.getRules()) {
        if (!inStreams.containsAll(rule.getStreams())) {
            throw new IllegalStateException("Input streams of rules processor " + inStreams + " does not contain rule's input streams " + rule.getStreams() + ". Please delete and recreate the rule.");
        }
        if (rule.getWindow() != null) {
            rulesWithWindow.add(rule);
        } else {
            rulesWithoutWindow.add(rule);
        }
    }
    // assert that RulesProcessor doesn't have mixed kinds of rules.
    if (!rulesWithWindow.isEmpty() && !rulesWithoutWindow.isEmpty()) {
        throw new IllegalStateException("RulesProcessor should have either windowed or non-windowed rules, not both.");
    }
    // associating multiple bolts to a rules processor is not allowed for simplicity.
    if (!rulesWithWindow.isEmpty()) {
        handleWindowedRules(rulesProcessor, rulesWithWindow);
    } else {
        // !rulesWithoutWindow.isEmpty()
        handleNonWindowedRules(rulesProcessor, rulesWithoutWindow);
    }
}

Example 12 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TopologyTestRunnerTest method runTest_withTestCaseId.

@Test
public void runTest_withTestCaseId() throws Exception {
    Topology topology = createSimpleDAGInjectedTestTopology();
    Long topologyId = topology.getId();
    Long testCaseId = 1L;
    TopologyTestRunCase testCase = new TopologyTestRunCase();
    testCase.setId(testCaseId);
    testCase.setTopologyId(topology.getId());
    testCase.setName("testcase1");
    testCase.setTimestamp(System.currentTimeMillis());
    setTopologyCurrentVersionExpectation(topology);
    setTopologyTestRunCaseExpectations(topology, testCase);
    setTopologyTestRunCaseSinkNotFoundExpectations(topology, testCase);
    setTopologyTestRunHistoryExpectations();
    setSucceedTopologyActionsExpectations();
    long sourceCount = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineSource).count();
    long sinkCount = topology.getTopologyDag().getInputComponents().stream().filter(c -> c instanceof StreamlineSink).count();
    TopologyTestRunHistory resultHistory = topologyTestRunner.runTest(topologyActions, topology, testCase, null);
    waitForTopologyTestRunToFinish(resultHistory);
    assertNotNull(resultHistory);
    assertTrue(resultHistory.getFinished());
    assertTrue(resultHistory.getSuccess());
    new VerificationsInOrder() {

        {
            catalogService.getTopologyTestRunCaseSourceBySourceId(testCaseId, anyLong);
            times = (int) sourceCount;
            catalogService.getTopologyTestRunCaseSinkBySinkId(testCaseId, anyLong);
            times = (int) sinkCount;
            TopologyTestRunHistory runHistory;
            // some fields are already modified after calling the method, so don't need to capture it
            catalogService.addTopologyTestRunHistory(withInstanceOf(TopologyTestRunHistory.class));
            times = 1;
            catalogService.addOrUpdateTopologyTestRunHistory(anyLong, runHistory = withCapture());
            times = 1;
            assertEquals(topology.getId(), runHistory.getTopologyId());
            assertEquals(topology.getVersionId(), runHistory.getVersionId());
            assertTrue(runHistory.getFinished());
            assertTrue(runHistory.getSuccess());
            assertNotNull(runHistory.getStartTime());
            assertNotNull(runHistory.getFinishTime());
            assertTrue(runHistory.getFinishTime() - runHistory.getStartTime() >= 0);
            assertTrue(isEmptyJson(runHistory.getExpectedOutputRecords()));
            assertTrue(isNotEmptyJson(runHistory.getActualOutputRecords()));
            assertFalse(runHistory.getMatched());
        }
    };
}

Example 13 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class NormalizationBoltTest method createFieldBasedNormalizationProcessor.

public static NormalizationProcessor createFieldBasedNormalizationProcessor(String outputStreamId) throws NormalizationException {
    Map<String, NormalizationConfig> inputStreamsWithConfig = new HashMap<>();
    Schema.Field tempField = new Schema.Field("temp", Schema.Type.INTEGER);
    Schema inputSchema = Schema.of(tempField, new Schema.Field("foo", Schema.Type.STRING));
    Transformer transformer = new Transformer(tempField, new Schema.Field("temperature", Schema.Type.FLOAT));
    transformer.setConverterScript("new Float((temp-32)*5/9f)");
    List<Transformer> transformers = Collections.singletonList(transformer);
    List<String> filters = Collections.singletonList("foo");
    List<FieldValueGenerator> fieldValueGenerators = Collections.singletonList(new FieldValueGenerator(new Schema.Field("new-field", Schema.Type.STRING), "new value"));
    FieldBasedNormalizationConfig fieldBasedNormalizationConfig = new FieldBasedNormalizationConfig(inputSchema, transformers, filters, fieldValueGenerators);
    inputStreamsWithConfig.put(INPUT_STREAM_ID, fieldBasedNormalizationConfig);
    Stream declaredOutputStream = new Stream(outputStreamId, OUTPUT_SCHEMA_FIELDS);
    NormalizationProcessor normalizationProcessor = new NormalizationProcessor(inputStreamsWithConfig, declaredOutputStream, NormalizationProcessor.Type.fineGrained);
    normalizationProcessor.addOutputStream(declaredOutputStream);
    return normalizationProcessor;
}

Example 14 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class NormalizationProcessorRuntime method initialize.

@Override
public void initialize(Map<String, Object> config) {
    final Iterator<Stream> iterator = normalizationProcessor.getOutputStreams().iterator();
    if (!iterator.hasNext()) {
        throw new IllegalStateException("normalization processor " + normalizationProcessor + " does not have output streams");
    }
    Stream outputStream = iterator.next();
    NormalizationRuntime.Factory factory = new NormalizationRuntime.Factory();
    Map<String, NormalizationRuntime> schemaRuntimes = new HashMap<>();
    for (Map.Entry<String, ? extends NormalizationConfig> entry : normalizationProcessor.getInputStreamsWithNormalizationConfig().entrySet()) {
        schemaRuntimes.put(entry.getKey(), factory.create(entry.getValue(), outputStream.getSchema(), normalizationProcessor.getNormalizationProcessorType()));
    }
    schemasWithNormalizationRuntime = schemaRuntimes;
    schemaValidator = new SchemaValidator(outputStream.getSchema());
}

Example 15 with Stream

use of com.hortonworks.streamline.streams.layout.component.Stream in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromProcessor.

@Test
public void visitProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    StreamlineProcessor originProcessor2 = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(originProcessor2);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, originProcessor2, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunProcessor testProcessor2 = createTestRunProcessor(originProcessor2);
    Map<String, TestRunProcessor> processorMap = new HashMap<>();
    processorMap.put(originProcessor.getName(), testProcessor);
    processorMap.put(originProcessor2.getName(), testProcessor2);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), processorMap, Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(originProcessor2);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunProcessor2Optional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor2.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunProcessor2Optional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunProcessor testRunProcessor2 = (TestRunProcessor) testRunProcessor2Optional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor2).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunProcessor2).get(0));
}