Examples with TopologyLayout com.hortonworks.streamline.streams.layout.component.TopologyLayout used on opensource projects

Example 6 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method testGetSinkComponentStatsWithoutAssigningTimeSeriesQuerier.

@Test
public void testGetSinkComponentStatsWithoutAssigningTimeSeriesQuerier() throws Exception {
    stormTopologyTimeSeriesMetrics.setTimeSeriesQuerier(null);
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final Map<String, Map<Long, Double>> expectedMisc = new HashMap<>();
    expectedMisc.put(StormMappedMetric.ackedRecords.name(), Collections.emptyMap());
    expectedMisc.put(StormMappedMetric.executeTime.name(), Collections.emptyMap());
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, sink, from, to, null);
    assertEquals(Collections.emptyMap(), actual.getInputRecords());
    assertEquals(Collections.emptyMap(), actual.getOutputRecords());
    assertEquals(Collections.emptyMap(), actual.getFailedRecords());
    assertEquals(Collections.emptyMap(), actual.getRecordsInWaitQueue());
    assertEquals(Collections.emptyMap(), actual.getProcessedTime());
    assertEquals(expectedMisc, actual.getMisc());
}

Example 7 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyActionsImpl method runTest.

@Override
public void runTest(TopologyLayout topology, TopologyTestRunHistory testRunHistory, String mavenArtifacts, Map<String, TestRunSource> testRunSourcesForEachSource, Map<String, TestRunProcessor> testRunProcessorsForEachProcessor, Map<String, TestRunRulesProcessor> testRunRulesProcessorsForEachProcessor, Map<String, TestRunSink> testRunSinksForEachSink, Optional<Long> durationSecs) throws Exception {
    TopologyDag originalTopologyDag = topology.getTopologyDag();
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originalTopologyDag, testRunSourcesForEachSource, testRunProcessorsForEachProcessor, testRunRulesProcessorsForEachProcessor, testRunSinksForEachSink);
    originalTopologyDag.traverse(visitor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    TopologyLayout testTopology = copyTopologyLayout(topology, testTopologyDag);
    Path jarToDeploy = addArtifactsToJar(getArtifactsLocation(testTopology));
    String fileName = createYamlFileForTest(testTopology);
    List<String> commands = new ArrayList<String>();
    commands.add(stormCliPath);
    commands.add("jar");
    commands.add(jarToDeploy.toString());
    commands.addAll(getExtraJarsArg(testTopology));
    commands.addAll(getMavenArtifactsRelatedArgs(mavenArtifacts));
    commands.addAll(getNimbusConf());
    commands.addAll(getForceNonSecuredClusterConf());
    commands.addAll(getTempWorkerArtifactArgs());
    commands.add("org.apache.storm.flux.Flux");
    commands.add("--local");
    commands.add("-s");
    if (durationSecs.isPresent()) {
        commands.add(String.valueOf(durationSecs.get() * 1000));
    } else {
        commands.add(String.valueOf(TEST_RUN_TOPOLOGY_DEFAULT_WAIT_MILLIS_FOR_SHUTDOWN));
    }
    commands.add(fileName);
    Process process = executeShellProcess(commands);
    ShellProcessResult shellProcessResult = waitTestRunProcess(process, testRunHistory.getId());
    int exitValue = shellProcessResult.exitValue;
    if (exitValue != 0) {
        LOG.error("Topology deploy command as test mode failed - exit code: {} / output: {}", exitValue, shellProcessResult.stdout);
        throw new Exception("Topology could not be run " + "successfully as test mode: storm deploy command failed");
    }
}

Example 8 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method getSinkComponentStats.

@Test
public void getSinkComponentStats() throws Exception {
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final TopologyTimeSeriesMetrics.TimeSeriesComponentMetric expectedMetric = setupExpectionToBoltComponent(sink, from, to);
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, sink, from, to, null);
    assertEquals(expectedMetric, actual);
}

Example 9 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class StormTopologyTimeSeriesMetricsImplTest method testGetSourceComponentStatsWithoutAssigningTimeSeriesQuerier.

@Test
public void testGetSourceComponentStatsWithoutAssigningTimeSeriesQuerier() throws Exception {
    stormTopologyTimeSeriesMetrics.setTimeSeriesQuerier(null);
    final TopologyLayout topology = getTopologyLayoutForTest();
    final long from = 1L;
    final long to = 3L;
    final Map<String, Map<Long, Double>> expectedMisc = new HashMap<>();
    expectedMisc.put(StormMappedMetric.ackedRecords.name(), Collections.emptyMap());
    expectedMisc.put(StormMappedMetric.completeLatency.name(), Collections.emptyMap());
    TopologyTimeSeriesMetrics.TimeSeriesComponentMetric actual = stormTopologyTimeSeriesMetrics.getComponentStats(topology, source, from, to, null);
    assertEquals(Collections.emptyMap(), actual.getInputRecords());
    assertEquals(Collections.emptyMap(), actual.getOutputRecords());
    assertEquals(Collections.emptyMap(), actual.getFailedRecords());
    assertEquals(Collections.emptyMap(), actual.getRecordsInWaitQueue());
    assertEquals(Collections.emptyMap(), actual.getProcessedTime());
    assertEquals(expectedMisc, actual.getMisc());
}

Example 10 with TopologyLayout

use of com.hortonworks.streamline.streams.layout.component.TopologyLayout in project streamline by hortonworks.

the class TopologyTestRunnerTest method setSucceedTopologyActionsExpectations.

private void setSucceedTopologyActionsExpectations() throws Exception {
    new Expectations() {

        {
            topologyActions.runTest(withInstanceOf(TopologyLayout.class), withInstanceOf(TopologyTestRunHistory.class), anyString, withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Map.class), withInstanceOf(Optional.class));
            result = new Delegate<Object>() {

                Object delegate(TopologyLayout topology, TopologyTestRunHistory testRunHistory, String mavenArtifacts, Map<String, TestRunSource> testRunSourcesForEachSource, Map<String, TestRunProcessor> testRunProcessorsForEachProcessor, Map<String, TestRunRulesProcessor> testRunRulesProcessorsForEachProcessor, Map<String, TestRunSink> testRunSinksForEachSink, Optional<Long> durationSecs) throws Exception {
                    Map<String, List<Map<String, Object>>> testOutputRecords = TopologyTestHelper.createTestOutputRecords(testRunSinksForEachSink.keySet());
                    testRunSinksForEachSink.entrySet().forEach(entry -> {
                        String sinkName = entry.getKey();
                        TestRunSink sink = entry.getValue();
                        try (FileWriter fw = new FileWriter(sink.getOutputFilePath())) {
                            List<Map<String, Object>> outputRecords = testOutputRecords.get(sinkName);
                            for (Map<String, Object> record : outputRecords) {
                                fw.write(objectMapper.writeValueAsString(record) + "\n");
                            }
                        } catch (IOException e) {
                            throw new RuntimeException(e);
                        }
                    });
                    return null;
                }
            };
        }
    };
}