Examples with TopologyTestRunCaseSource com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource used on opensource projects

Example 11 with TopologyTestRunCaseSource

use of com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource in project streamline by hortonworks.

the class StreamCatalogService method getTopologyTestRunCaseSource.

public TopologyTestRunCaseSource getTopologyTestRunCaseSource(Long testcaseId, Long id) {
    TopologyTestRunCaseSource testCaseSource = new TopologyTestRunCaseSource();
    testCaseSource.setId(id);
    TopologyTestRunCaseSource retrieved = dao.get(new StorableKey(TopologyTestRunCaseSource.NAMESPACE, testCaseSource.getPrimaryKey()));
    if (retrieved == null || !retrieved.getTestCaseId().equals(testcaseId)) {
        return null;
    }
    return retrieved;
}

Example 12 with TopologyTestRunCaseSource

use of com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource in project streamline by hortonworks.

the class StreamCatalogService method removeTopologyDependencies.

private void removeTopologyDependencies(Long topologyId, Long versionId) throws Exception {
    List<QueryParam> topologyIdVersionIdQueryParams = WSUtils.buildTopologyIdAndVersionIdAwareQueryParams(topologyId, versionId, null);
    // remove topology test histories
    Collection<TopologyTestRunHistory> runHistories = listTopologyTestRunHistory(topologyId, versionId);
    runHistories.forEach(history -> removeTopologyTestRunHistory(history.getId()));
    // remove topology test run case
    Collection<TopologyTestRunCase> runCases = listTopologyTestRunCase(topologyIdVersionIdQueryParams);
    for (TopologyTestRunCase runCase : runCases) {
        Collection<TopologyTestRunCaseSource> runCaseSources = listTopologyTestRunCaseSource(runCase.getId());
        Collection<TopologyTestRunCaseSink> runCaseSinks = listTopologyTestRunCaseSink(runCase.getId());
        // remove topology test run case source
        for (TopologyTestRunCaseSource runCaseSource : runCaseSources) {
            removeTopologyTestRunCaseSource(runCaseSource.getId());
        }
        // remove topology test run case sink
        for (TopologyTestRunCaseSink runCaseSink : runCaseSinks) {
            removeTopologyTestRunCaseSink(runCaseSink.getId());
        }
        removeTestRunCase(topologyId, runCase.getId());
    }
    // remove edges
    Collection<TopologyEdge> edges = listTopologyEdges(topologyIdVersionIdQueryParams);
    for (TopologyEdge edge : edges) {
        removeTopologyEdge(topologyId, edge.getId(), versionId);
    }
    // remove rules
    Collection<TopologyRule> topologyRules = listRules(topologyIdVersionIdQueryParams);
    for (TopologyRule topologyRule : topologyRules) {
        removeRule(topologyId, topologyRule.getId(), versionId);
    }
    // remove windowed rules
    Collection<TopologyWindow> topologyWindows = listWindows(topologyIdVersionIdQueryParams);
    for (TopologyWindow topologyWindow : topologyWindows) {
        removeWindow(topologyId, topologyWindow.getId(), versionId);
    }
    // remove branch rules
    Collection<TopologyBranchRule> topologyBranchRules = listBranchRules(topologyIdVersionIdQueryParams);
    for (TopologyBranchRule topologyBranchRule : topologyBranchRules) {
        removeBranchRule(topologyId, topologyBranchRule.getId(), versionId);
    }
    // remove sinks
    Collection<TopologySink> sinks = listTopologySinks(topologyIdVersionIdQueryParams);
    for (TopologySink sink : sinks) {
        removeTopologySink(topologyId, sink.getId(), versionId, false);
    }
    // remove processors
    Collection<TopologyProcessor> processors = listTopologyProcessors(topologyIdVersionIdQueryParams);
    for (TopologyProcessor processor : processors) {
        removeTopologyProcessor(topologyId, processor.getId(), versionId, false);
    }
    // remove sources
    Collection<TopologySource> sources = listTopologySources(topologyIdVersionIdQueryParams);
    for (TopologySource source : sources) {
        removeTopologySource(topologyId, source.getId(), versionId, false);
    }
    // remove output streams
    Collection<TopologyStream> topologyStreams = listStreamInfos(topologyIdVersionIdQueryParams);
    for (TopologyStream topologyStream : topologyStreams) {
        removeStreamInfo(topologyId, topologyStream.getId(), versionId);
    }
    // remove topology editor metadata
    removeTopologyEditorMetadata(topologyId, versionId);
}

Example 13 with TopologyTestRunCaseSource

use of com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource in project streamline by hortonworks.

the class StreamCatalogService method removeTopologyTestRunCaseSource.

public TopologyTestRunCaseSource removeTopologyTestRunCaseSource(Long id) {
    TopologyTestRunCaseSource testcaseSource = new TopologyTestRunCaseSource();
    testcaseSource.setId(id);
    return dao.remove(testcaseSource.getStorableKey());
}

Example 14 with TopologyTestRunCaseSource

use of com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource in project streamline by hortonworks.

the class TopologyTestRunnerTest method createTopologyTestRunCaseSource.

private TopologyTestRunCaseSource createTopologyTestRunCaseSource(TopologyTestRunCase testRunCase, StreamlineSource source) {
    TopologyTestRunCaseSource testRunSource = new TopologyTestRunCaseSource();
    testRunSource.setId(Long.valueOf(source.getId()));
    testRunSource.setTestCaseId(testRunCase.getId());
    testRunSource.setSourceId(Long.valueOf(source.getId()));
    try {
        Map<String, String> testRecords = Collections.singletonMap("default", objectMapper.writeValueAsString(TopologyTestHelper.createTestRecords()));
        testRunSource.setRecords(objectMapper.writeValueAsString(testRecords));
    } catch (JsonProcessingException e) {
        throw new RuntimeException("Can't serialize test records map into JSON");
    }
    testRunSource.setOccurrence(1);
    testRunSource.setSleepMsPerIteration(0L);
    testRunSource.setTimestamp(System.currentTimeMillis());
    return testRunSource;
}

Example 15 with TopologyTestRunCaseSource

use of com.hortonworks.streamline.streams.catalog.TopologyTestRunCaseSource in project streamline by hortonworks.

the class TopologyTestRunner method runTest.

public TopologyTestRunHistory runTest(TopologyActions topologyActions, Topology topology, TopologyTestRunCase testCase, Long durationSecs) throws IOException {
    List<StreamlineSource> sources = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineSource).map(c -> (StreamlineSource) c).collect(toList());
    List<StreamlineSink> sinks = topology.getTopologyDag().getInputComponents().stream().filter(c -> c instanceof StreamlineSink).map(c -> (StreamlineSink) c).collect(toList());
    List<StreamlineProcessor> processors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof StreamlineProcessor && !(c instanceof RulesProcessor)).map(c -> (StreamlineProcessor) c).collect(toList());
    List<RulesProcessor> rulesProcessors = topology.getTopologyDag().getOutputComponents().stream().filter(c -> c instanceof RulesProcessor).map(c -> (RulesProcessor) c).collect(toList());
    // load test case sources for all sources
    List<TopologyTestRunCaseSource> testRunCaseSources = sources.stream().map(s -> catalogService.getTopologyTestRunCaseSourceBySourceId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    if (testRunCaseSources.stream().anyMatch(Objects::isNull)) {
        throw new IllegalArgumentException("Not every source register test records.");
    }
    // load test case sources for all sinks
    List<TopologyTestRunCaseSink> testRunCaseSinks = sinks.stream().map(s -> catalogService.getTopologyTestRunCaseSinkBySinkId(testCase.getId(), Long.valueOf(s.getId()))).collect(toList());
    Map<Long, Map<String, List<Map<String, Object>>>> testRecordsForEachSources = readTestRecordsFromTestCaseSources(testRunCaseSources);
    Map<Long, Integer> occurrenceForEachSources = readOccurrenceFromTestCaseSources(testRunCaseSources);
    Map<Long, Long> sleepMsPerRecordsForEachSources = readSleepMsPerIterationFromTestCaseSources(testRunCaseSources);
    Map<String, List<Map<String, Object>>> expectedOutputRecordsMap = readExpectedRecordsFromTestCaseSinks(sinks, testRunCaseSinks);
    String eventLogFilePath = getTopologyTestRunEventLog(topology);
    Map<String, TestRunSource> testRunSourceMap = sources.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunSource testRunSource = new TestRunSource(s.getOutputStreams(), testRecordsForEachSources.get(Long.valueOf(s.getId())), occurrenceForEachSources.get(Long.valueOf(s.getId())), sleepMsPerRecordsForEachSources.get(Long.valueOf(s.getId())), eventLogFilePath);
        testRunSource.setName(s.getName());
        return testRunSource;
    }));
    Map<String, TestRunSink> testRunSinkMap = sinks.stream().collect(toMap(s -> s.getName(), s -> {
        String uuid = UUID.randomUUID().toString();
        TestRunSink testRunSink = new TestRunSink(getTopologyTestRunResult(uuid));
        testRunSink.setName(s.getName());
        return testRunSink;
    }));
    Map<String, TestRunProcessor> testRunProcessorMap = processors.stream().collect(toMap(s -> s.getName(), s -> {
        if (s instanceof JoinProcessor) {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, true, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        } else {
            TestRunProcessor testRunProcessor = new TestRunProcessor(s, false, eventLogFilePath);
            testRunProcessor.setName(s.getName());
            return testRunProcessor;
        }
    }));
    Map<String, TestRunRulesProcessor> testRunRulesProcessorMap = rulesProcessors.stream().collect(toMap(s -> s.getName(), s -> {
        TestRunRulesProcessor testRunRulesProcessor = new TestRunRulesProcessor(s, eventLogFilePath);
        testRunRulesProcessor.setName(s.getName());
        return testRunRulesProcessor;
    }));
    // just create event file before running actual test run process
    createEventLogFile(eventLogFilePath);
    TopologyTestRunHistory history = initializeTopologyTestRunHistory(topology, testCase, expectedOutputRecordsMap, eventLogFilePath);
    catalogService.addTopologyTestRunHistory(history);
    Optional<Long> finalDurationSecs = Optional.ofNullable(durationSecs);
    ParallelStreamUtil.runAsync(() -> runTestInBackground(topologyActions, topology, history, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, expectedOutputRecordsMap, finalDurationSecs), forkJoinPool);
    return history;
}