Examples with TopologyLayout com.hortonworks.streamline.streams.layout.component.TopologyLayout used on opensource projects

Example 1 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class TopologyTestRunner method runTestInBackground.

private Void runTestInBackground(TopologyActions topologyActions, Topology topology, TopologyTestRunHistory history, Map<String, TestRunSource> testRunSourceMap, Map<String, TestRunProcessor> testRunProcessorMap, Map<String, TestRunRulesProcessor> testRunRulesProcessorMap, Map<String, TestRunSink> testRunSinkMap, Map<String, List<Map<String, Object>>> expectedOutputRecordsMap, Optional<Long> durationSecs) throws IOException {
    TopologyLayout topologyLayout = CatalogToLayoutConverter.getTopologyLayout(topology, topology.getTopologyDag());
    try {
        topologyActionsService.setUpClusterArtifacts(topology, topologyActions);
        String mavenArtifacts = topologyActionsService.setUpExtraJars(topology, topologyActions);
        topologyActions.runTest(topologyLayout, history, mavenArtifacts, testRunSourceMap, testRunProcessorMap, testRunRulesProcessorMap, testRunSinkMap, durationSecs);
        history.finishSuccessfully();
        Map<String, List<Map<String, Object>>> actualOutputRecordsMap = parseTestRunOutputFiles(testRunSinkMap);
        history.setActualOutputRecords(objectMapper.writeValueAsString(actualOutputRecordsMap));
        if (expectedOutputRecordsMap != null) {
            boolean matched = equalsOutputRecords(expectedOutputRecordsMap, actualOutputRecordsMap);
            history.setMatched(matched);
        }
    } catch (Throwable e) {
        LOG.warn("Exception thrown while running Topology as test mode. Marking as 'failed'. topology id: {}", topology.getId(), e);
        history.finishWithFailures();
    }
    catalogService.addOrUpdateTopologyTestRunHistory(history.getId(), history);
    return null;
}

Example 2 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyActionsImpl method deploy.

@Override
public void deploy(TopologyLayout topology, String mavenArtifacts, TopologyActionContext ctx, String asUser) throws Exception {
    ctx.setCurrentAction("Adding artifacts to jar");
    Path jarToDeploy = addArtifactsToJar(getArtifactsLocation(topology));
    ctx.setCurrentAction("Creating Storm topology YAML file");
    String fileName = createYamlFileForDeploy(topology);
    ctx.setCurrentAction("Deploying topology via 'storm jar' command");
    List<String> commands = new ArrayList<String>();
    commands.add(stormCliPath);
    commands.add("jar");
    commands.add(jarToDeploy.toString());
    commands.addAll(getExtraJarsArg(topology));
    commands.addAll(getMavenArtifactsRelatedArgs(mavenArtifacts));
    commands.addAll(getNimbusConf());
    commands.addAll(getSecuredClusterConf(asUser));
    commands.add("org.apache.storm.flux.Flux");
    commands.add("--remote");
    commands.add(fileName);
    LOG.info("Deploying Application {}", topology.getName());
    LOG.info(String.join(" ", commands));
    Process process = executeShellProcess(commands);
    ShellProcessResult shellProcessResult = waitProcessFor(process);
    int exitValue = shellProcessResult.exitValue;
    if (exitValue != 0) {
        LOG.error("Topology deploy command failed - exit code: {} / output: {}", exitValue, shellProcessResult.stdout);
        String[] lines = shellProcessResult.stdout.split("\\n");
        String errors = Arrays.stream(lines).filter(line -> line.startsWith("Exception") || line.startsWith("Caused by")).collect(Collectors.joining(", "));
        Pattern pattern = Pattern.compile("Topology with name `(.*)` already exists on cluster");
        Matcher matcher = pattern.matcher(errors);
        if (matcher.find()) {
            throw new TopologyAlreadyExistsOnCluster(matcher.group(1));
        } else {
            throw new Exception("Topology could not be deployed successfully: storm deploy command failed with " + errors);
        }
    }
}

Example 3 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method testGetProcessorComponentStatsWithoutAssigningTimeSeriesQuerier.

@Test
public void testGetProcessorComponentStatsWithoutAssigningTimeSeriesQuerier() throws Exception {
    stormTopologyTimeSeriesMetrics.setTimeSeriesQuerier(null);
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final Map<String, Map<Long, Double>> expectedMisc = new HashMap<>();
    expectedMisc.put(StormMappedMetric.ackedRecords.name(), Collections.emptyMap());
    expectedMisc.put(StormMappedMetric.executeTime.name(), Collections.emptyMap());
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, processor, from, to, null);
    assertEquals(Collections.emptyMap(), actual.getInputRecords());
    assertEquals(Collections.emptyMap(), actual.getOutputRecords());
    assertEquals(Collections.emptyMap(), actual.getFailedRecords());
    assertEquals(Collections.emptyMap(), actual.getRecordsInWaitQueue());
    assertEquals(Collections.emptyMap(), actual.getProcessedTime());
    assertEquals(expectedMisc, actual.getMisc());
}

Example 4 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method getProcessorComponentStats.

@Test
public void getProcessorComponentStats() throws Exception {
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final TopologyTimeSeriesMetrics.TimeSeriesComponentMetric expectedMetric = setupExpectionToBoltComponent(processor, from, to);
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, processor, from, to, null);
    assertEquals(expectedMetric, actual);
}

Example 5 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method getSourceComponentStats.

@Test
public void getSourceComponentStats() throws Exception {
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final TopologyTimeSeriesMetrics.TimeSeriesComponentMetric expectedMetric = setupExpectionToSpoutComponent(source, from, to);
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, source, from, to, null);
    assertEquals(expectedMetric, actual);
}