Examples with TestRunProcessor com.hortonworks.streamline.streams.layout.component.impl.testing.TestRunProcessor used on opensource projects

Example 6 with TestRunProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.testing.TestRunProcessor in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitRulesProcessor_connectedFromProcessor.

@Test
public void visitRulesProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    RulesProcessor rulesProcessor = TopologyTestHelper.createRulesProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(rulesProcessor);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, rulesProcessor, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunRulesProcessor testRulesProcessor = createTestRunRulesProcessor(rulesProcessor);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), Collections.singletonMap(originProcessor.getName(), testProcessor), Collections.singletonMap(rulesProcessor.getName(), testRulesProcessor), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(rulesProcessor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    List<OutputComponent> testRulesProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunRulesProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunRuleProcessorOptional = testRulesProcessors.stream().filter(o -> o.getName().equals(rulesProcessor.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunRuleProcessorOptional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunRulesProcessor testRunRuleProcessor = (TestRunRulesProcessor) testRunRuleProcessorOptional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunRuleProcessor).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunRuleProcessor).get(0));
}

Example 7 with TestRunProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.testing.TestRunProcessor in project streamline by hortonworks.

the class TopologyTestRunnerTest method setSucceedTopologyActionsExpectations.

private void setSucceedTopologyActionsExpectations() throws Exception {
    new Expectations() {

        {
            topologyActions.runTest(withInstanceOf(TopologyLayout.class), withInstanceOf(TopologyTestRunHistory.class), anyString, withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Optional.class));
            result = new Delegate<Object>() {

                Object delegate(TopologyLayout topology, TopologyTestRunHistory testRunHistory, String mavenArtifacts, Map<String, TestRunSource> testRunSourcesForEachSource, Map<String, TestRunProcessor> testRunProcessorsForEachProcessor, Map<String, TestRunRulesProcessor> testRunRulesProcessorsForEachProcessor, Map<String, TestRunSink> testRunSinksForEachSink, Optional<Long> durationSecs) throws Exception {
                    Map<String, List<Map<String, Object>>> testOutputRecords = TopologyTestHelper.createTestOutputRecords(testRunSinksForEachSink.keySet());
                    testRunSinksForEachSink.entrySet().forEach(entry -> {
                        String sinkName = entry.getKey();
                        TestRunSink sink = entry.getValue();
                        try (FileWriter fw = new FileWriter(sink.getOutputFilePath())) {
                            List<Map<String, Object>> outputRecords = testOutputRecords.get(sinkName);
                            for (Map<String, Object> record : outputRecords) {
                                fw.write(objectMapper.writeValueAsString(record) + "\n");
                            }
                        } catch (IOException e) {
                            throw new RuntimeException(e);
                        }
                    });
                    return null;
                }
            };
        }
    };
}

Example 8 with TestRunProcessor

use of com.hortonworks.streamline.streams.layout.component.impl.testing.TestRunProcessor in project streamline by hortonworks.

the class TopologyTestRunner method runTest.

public TopologyTestRunHistory runTest(TopologyActions topologyActions, Topology topology, TopologyTestRunCase testCase, Long durationSecs) throws IOException {
    List<StreamlineSource> sources = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineSource).map(c -> (StreamlineSource) c).collect(toList());
    List<StreamlineSink> sinks = topology.getTopologyDag().getInputComponents().stream().filter(c -> c instanceof StreamlineSink).map(c -> (StreamlineSink) c).collect(toList());
    List<StreamlineProcessor> processors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineProcessor && !(c instanceof RulesProcessor)).map(c -> (StreamlineProcessor) c).collect(toList());
    List<RulesProcessor> rulesProcessors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof RulesProcessor).map(c -> (RulesProcessor) c).collect(toList());
    // load test case sources for all sources
    List<TopologyTestRunCaseSource> testRunCaseSources = sources.stream().map(s -> catalogService.getTopologyTestRunCaseSourceBySourceId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    if (testRunCaseSources.stream().anyMatch(Objects::isNull)) {
        throw new IllegalArgumentException("Not every source register test records.");
    }
    // load test case sources for all sinks
    List<TopologyTestRunCaseSink> testRunCaseSinks = sinks.stream().map(s -> catalogService.getTopologyTestRunCaseSinkBySinkId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    Map<Long, Map<String, List<Map<String, Object>>>> testRecordsForEachSources = readTestRecordsFromTestCaseSources(testRunCaseSources);
    Map<Long, Integer> occurrenceForEachSources = readOccurrenceFromTestCaseSources(testRunCaseSources);
    Map<Long, Long> sleepMsPerRecordsForEachSources = readSleepMsPerIterationFromTestCaseSources(testRunCaseSources);
    Map<String, List<Map<String, Object>>> expectedOutputRecordsMap = readExpectedRecordsFromTestCaseSinks(sinks, testRunCaseSinks);
    String eventLogFilePath = getTopologyTestRunEventLog(topology);
    Map<String, TestRunSource> testRunSourceMap = sources.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunSource testRunSource = new TestRunSource(s.getOutputStreams(), testRecordsForEachSources.get(Long.valueOf(s.getId())), occurrenceForEachSources.get(Long.valueOf(s.getId())), sleepMsPerRecordsForEachSources.get(Long.valueOf(s.getId())), eventLogFilePath);
        testRunSource.setName(s.getName());
        return testRunSource;
    }));
    Map<String, TestRunSink> testRunSinkMap = sinks.stream().collect(toMap(s -> s.getName(), s -> {
        String uuid = UUID.randomUUID().toString();
        TestRunSink testRunSink = new TestRunSink(getTopologyTestRunResult(uuid));
        testRunSink.setName(s.getName());
        return testRunSink;
    }));
    Map<String, TestRunProcessor> testRunProcessorMap = processors.stream().collect(toMap(s -> s.getName(), s -> {
        if (s instanceof JoinProcessor) {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, true, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        } else {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, false, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        }
    }));
    Map<String, TestRunRulesProcessor> testRunRulesProcessorMap = rulesProcessors.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunRulesProcessor testRunRulesProcessor = new TestRunRulesProcessor(s, eventLogFilePath);
        testRunRulesProcessor.setName(s.getName());
        return testRunRulesProcessor;
    }));
    // just create event file before running actual test run process
    createEventLogFile(eventLogFilePath);
    TopologyTestRunHistory history = initializeTopologyTestRunHistory(topology, testCase, expectedOutputRecordsMap, eventLogFilePath);
    catalogService.addTopologyTestRunHistory(history);
    Optional<Long> finalDurationSecs = Optional.ofNullable(durationSecs);
    ParallelStreamUtil.runAsync(() -> runTestInBackground(topologyActions, topology, history, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, expectedOutputRecordsMap, finalDurationSecs), forkJoinPool);
    return history;
}