Examples with StreamlineSource com.hortonworks.streamline.streams.layout.component.StreamlineSource used on opensource projects

Example 11 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitSink_connectedFromSource.

@Test
public void visitSink_connectedFromSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    StreamlineSink originSink = TopologyTestHelper.createStreamlineSink("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    originTopologyDag.add(originSink);
    originTopologyDag.addEdge(new Edge("e1", originSource, originSink, "default", Stream.Grouping.SHUFFLE));
    TestRunSource testSource = createTestRunSource(originSource);
    TestRunSink testSink = createTestRunSink(originSink);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.emptyMap(), Collections.emptyMap(), Collections.singletonMap(originSink.getName(), testSink));
    visitor.visit(originSource);
    visitor.visit(originSink);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    List<InputComponent> testSinks = testTopologyDag.getInputComponents().stream().filter(o -> (o instanceof TestRunSink && o.getName().equals(originSink.getName()))).collect(toList());
    assertEquals(1, testSinks.size());
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    TestRunSink testRunSink = (TestRunSink) testSinks.get(0);
    assertEquals(originSink.getId(), testRunSink.getId());
    assertEquals(testSink.getOutputFilePath(), testRunSink.getOutputFilePath());
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunSource).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunSink).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunSource).get(0) == testTopologyDag.getEdgesTo(testRunSink).get(0));
}

Example 12 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TopologyTestHelper method createStreamlineSource.

public static StreamlineSource createStreamlineSource(String id) {
    Stream stream = createDefaultStream();
    StreamlineSource source = new StreamlineSource(Sets.newHashSet(stream));
    source.setId(id);
    source.setName("testSource_" + id);
    source.setConfig(new Config());
    source.setTransformationClass("dummyTransformation");
    return source;
}

Example 13 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TopologyComponentFactory method getStreamlineSource.

public StreamlineSource getStreamlineSource(TopologySource topologySource) {
    StreamlineSource source = getStreamlineComponent(StreamlineSource.class, topologySource);
    source.addOutputStreams(createOutputStreams(topologySource));
    return source;
}

Example 14 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TopologyTestRunnerTest method runTest_topologyActionsTestRunFails.

@Test
public void runTest_topologyActionsTestRunFails() throws Exception {
    Topology topology = createSimpleDAGInjectedTestTopology();
    Long topologyId = topology.getId();
    Long testCaseId = 1L;
    TopologyTestRunCase testCase = new TopologyTestRunCase();
    testCase.setId(testCaseId);
    testCase.setTopologyId(topology.getId());
    testCase.setName("testcase1");
    testCase.setTimestamp(System.currentTimeMillis());
    setTopologyCurrentVersionExpectation(topology);
    setTopologyTestRunCaseExpectations(topology, testCase);
    setTopologyTestRunCaseSinkNotFoundExpectations(topology, testCase);
    setTopologyTestRunHistoryExpectations();
    setTopologyActionsThrowingExceptionExpectations();
    long sourceCount = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineSource).count();
    long sinkCount = topology.getTopologyDag().getInputComponents().stream().filter(c -> c instanceof StreamlineSink).count();
    TopologyTestRunHistory resultHistory = topologyTestRunner.runTest(topologyActions, topology, testCase, null);
    assertNotNull(resultHistory);
    waitForTopologyTestRunToFinish(resultHistory);
    new VerificationsInOrder() {

        {
            catalogService.getTopologyTestRunCaseSourceBySourceId(testCaseId, anyLong);
            times = (int) sourceCount;
            catalogService.getTopologyTestRunCaseSinkBySinkId(testCaseId, anyLong);
            times = (int) sinkCount;
            TopologyTestRunHistory runHistory;
            // some fields are already modified after calling the method, so don't need to capture it
            catalogService.addTopologyTestRunHistory(withInstanceOf(TopologyTestRunHistory.class));
            times = 1;
            catalogService.addOrUpdateTopologyTestRunHistory(anyLong, runHistory = withCapture());
            times = 1;
            assertEquals(topology.getId(), runHistory.getTopologyId());
            assertEquals(topology.getVersionId(), runHistory.getVersionId());
            assertTrue(runHistory.getFinished());
            assertFalse(runHistory.getSuccess());
            assertNotNull(runHistory.getStartTime());
            assertNotNull(runHistory.getFinishTime());
            assertTrue(runHistory.getFinishTime() - runHistory.getStartTime() >= 0);
            assertTrue(isEmptyJson(runHistory.getExpectedOutputRecords()));
            assertNull(runHistory.getActualOutputRecords());
            assertFalse(runHistory.getMatched());
        }
    };
}

Example 15 with StreamlineSource

use of com.hortonworks.streamline.streams.layout.component.StreamlineSource in project streamline by hortonworks.

the class TopologyTestRunner method runTest.

public TopologyTestRunHistory runTest(TopologyActions topologyActions, Topology topology, TopologyTestRunCase testCase, Long durationSecs) throws IOException {
    List<StreamlineSource> sources = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineSource).map(c -> (StreamlineSource) c).collect(toList());
    List<StreamlineSink> sinks = topology.getTopologyDag().getInputComponents().stream().filter(c -> c instanceof StreamlineSink).map(c -> (StreamlineSink) c).collect(toList());
    List<StreamlineProcessor> processors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineProcessor && !(c instanceof RulesProcessor)).map(c -> (StreamlineProcessor) c).collect(toList());
    List<RulesProcessor> rulesProcessors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof RulesProcessor).map(c -> (RulesProcessor) c).collect(toList());
    // load test case sources for all sources
    List<TopologyTestRunCaseSource> testRunCaseSources = sources.stream().map(s -> catalogService.getTopologyTestRunCaseSourceBySourceId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    if (testRunCaseSources.stream().anyMatch(Objects::isNull)) {
        throw new IllegalArgumentException("Not every source register test records.");
    }
    // load test case sources for all sinks
    List<TopologyTestRunCaseSink> testRunCaseSinks = sinks.stream().map(s -> catalogService.getTopologyTestRunCaseSinkBySinkId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    Map<Long, Map<String, List<Map<String, Object>>>> testRecordsForEachSources = readTestRecordsFromTestCaseSources(testRunCaseSources);
    Map<Long, Integer> occurrenceForEachSources = readOccurrenceFromTestCaseSources(testRunCaseSources);
    Map<Long, Long> sleepMsPerRecordsForEachSources = readSleepMsPerIterationFromTestCaseSources(testRunCaseSources);
    Map<String, List<Map<String, Object>>> expectedOutputRecordsMap = readExpectedRecordsFromTestCaseSinks(sinks, testRunCaseSinks);
    String eventLogFilePath = getTopologyTestRunEventLog(topology);
    Map<String, TestRunSource> testRunSourceMap = sources.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunSource testRunSource = new TestRunSource(s.getOutputStreams(), testRecordsForEachSources.get(Long.valueOf(s.getId())), occurrenceForEachSources.get(Long.valueOf(s.getId())), sleepMsPerRecordsForEachSources.get(Long.valueOf(s.getId())), eventLogFilePath);
        testRunSource.setName(s.getName());
        return testRunSource;
    }));
    Map<String, TestRunSink> testRunSinkMap = sinks.stream().collect(toMap(s -> s.getName(), s -> {
        String uuid = UUID.randomUUID().toString();
        TestRunSink testRunSink = new TestRunSink(getTopologyTestRunResult(uuid));
        testRunSink.setName(s.getName());
        return testRunSink;
    }));
    Map<String, TestRunProcessor> testRunProcessorMap = processors.stream().collect(toMap(s -> s.getName(), s -> {
        if (s instanceof JoinProcessor) {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, true, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        } else {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, false, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        }
    }));
    Map<String, TestRunRulesProcessor> testRunRulesProcessorMap = rulesProcessors.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunRulesProcessor testRunRulesProcessor = new TestRunRulesProcessor(s, eventLogFilePath);
        testRunRulesProcessor.setName(s.getName());
        return testRunRulesProcessor;
    }));
    // just create event file before running actual test run process
    createEventLogFile(eventLogFilePath);
    TopologyTestRunHistory history = initializeTopologyTestRunHistory(topology, testCase, expectedOutputRecordsMap, eventLogFilePath);
    catalogService.addTopologyTestRunHistory(history);
    Optional<Long> finalDurationSecs = Optional.ofNullable(durationSecs);
    ParallelStreamUtil.runAsync(() -> runTestInBackground(topologyActions, topology, history, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, expectedOutputRecordsMap, finalDurationSecs), forkJoinPool);
    return history;
}