Examples with TopologyDag com.hortonworks.streamline.streams.layout.component.TopologyDag used on opensource projects

Example 6 with TopologyDag

use of com.hortonworks.streamline.streams.layout.component.TopologyDag in project streamline by hortonworks.

the class TopologyTestRunnerTest method injectTestTopology.

private void injectTestTopology(Topology topology) {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("2");
    StreamlineSink originSink = TopologyTestHelper.createStreamlineSink("3");
    TopologyDag topologyDag = new TopologyDag();
    topologyDag.add(originSource);
    topologyDag.add(originProcessor);
    topologyDag.add(originSink);
    topologyDag.addEdge(new Edge("e1", originSource, originProcessor, "default", Stream.Grouping.SHUFFLE));
    topologyDag.addEdge(new Edge("e2", originProcessor, originSink, "default", Stream.Grouping.SHUFFLE));
    topology.setTopologyDag(topologyDag);
}

Example 7 with TopologyDag

use of com.hortonworks.streamline.streams.layout.component.TopologyDag in project streamline by hortonworks.

the class StormTopologyActionsImpl method createYamlFile.

private String createYamlFile(TopologyLayout topology, boolean deploy) throws Exception {
    Map<String, Object> yamlMap;
    File f;
    OutputStreamWriter fileWriter = null;
    try {
        f = new File(this.getFilePath(topology));
        if (f.exists()) {
            if (!f.delete()) {
                throw new Exception("Unable to delete old storm " + "artifact for topology id " + topology.getId());
            }
        }
        yamlMap = new LinkedHashMap<>();
        yamlMap.put(StormTopologyLayoutConstants.YAML_KEY_NAME, generateStormTopologyName(topology));
        TopologyDag topologyDag = topology.getTopologyDag();
        LOG.debug("Initial Topology config {}", topology.getConfig());
        StormTopologyFluxGenerator fluxGenerator = new StormTopologyFluxGenerator(topology, conf, getExtraJarsLocation(topology));
        topologyDag.traverse(fluxGenerator);
        for (Map.Entry<String, Map<String, Object>> entry : fluxGenerator.getYamlKeysAndComponents()) {
            addComponentToCollection(yamlMap, entry.getValue(), entry.getKey());
        }
        Config topologyConfig = fluxGenerator.getTopologyConfig();
        putAutoTokenDelegationConfig(topologyConfig, topologyDag);
        registerEventLogger(topologyConfig);
        Map<String, Object> properties = topologyConfig.getProperties();
        if (!deploy) {
            LOG.debug("Disabling topology event logger for test mode...");
            properties.put("topology.eventlogger.executors", 0);
        }
        LOG.debug("Final Topology properties {}", properties);
        addTopologyConfig(yamlMap, properties);
        DumperOptions options = new DumperOptions();
        options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
        options.setSplitLines(false);
        // options.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN);
        Yaml yaml = new Yaml(options);
        fileWriter = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
        yaml.dump(yamlMap, fileWriter);
        return f.getAbsolutePath();
    } finally {
        if (fileWriter != null) {
            fileWriter.close();
        }
    }
}

Example 8 with TopologyDag

use of com.hortonworks.streamline.streams.layout.component.TopologyDag in project streamline by hortonworks.

the class StormTopologyActionsImpl method runTest.

@Override
public void runTest(TopologyLayout topology, TopologyTestRunHistory testRunHistory, String mavenArtifacts, Map<String, TestRunSource> testRunSourcesForEachSource, Map<String, TestRunProcessor> testRunProcessorsForEachProcessor, Map<String, TestRunRulesProcessor> testRunRulesProcessorsForEachProcessor, Map<String, TestRunSink> testRunSinksForEachSink, Optional<Long> durationSecs) throws Exception {
    TopologyDag originalTopologyDag = topology.getTopologyDag();
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originalTopologyDag, testRunSourcesForEachSource, testRunProcessorsForEachProcessor, testRunRulesProcessorsForEachProcessor, testRunSinksForEachSink);
    originalTopologyDag.traverse(visitor);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    TopologyLayout testTopology = copyTopologyLayout(topology, testTopologyDag);
    Path jarToDeploy = addArtifactsToJar(getArtifactsLocation(testTopology));
    String fileName = createYamlFileForTest(testTopology);
    List<String> commands = new ArrayList<String>();
    commands.add(stormCliPath);
    commands.add("jar");
    commands.add(jarToDeploy.toString());
    commands.addAll(getExtraJarsArg(testTopology));
    commands.addAll(getMavenArtifactsRelatedArgs(mavenArtifacts));
    commands.addAll(getNimbusConf());
    commands.addAll(getForceNonSecuredClusterConf());
    commands.addAll(getTempWorkerArtifactArgs());
    commands.add("org.apache.storm.flux.Flux");
    commands.add("--local");
    commands.add("-s");
    if (durationSecs.isPresent()) {
        commands.add(String.valueOf(durationSecs.get() * 1000));
    } else {
        commands.add(String.valueOf(TEST_RUN_TOPOLOGY_DEFAULT_WAIT_MILLIS_FOR_SHUTDOWN));
    }
    commands.add(fileName);
    Process process = executeShellProcess(commands);
    ShellProcessResult shellProcessResult = waitTestRunProcess(process, testRunHistory.getId());
    int exitValue = shellProcessResult.exitValue;
    if (exitValue != 0) {
        LOG.error("Topology deploy command as test mode failed - exit code: {} / output: {}", exitValue, shellProcessResult.stdout);
        throw new Exception("Topology could not be run " + "successfully as test mode: storm deploy command failed");
    }
}

Example 9 with TopologyDag

use of com.hortonworks.streamline.streams.layout.component.TopologyDag in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitProcessor_connectedFromProcessor.

@Test
public void visitProcessor_connectedFromProcessor() throws Exception {
    StreamlineProcessor originProcessor = TopologyTestHelper.createStreamlineProcessor("1");
    StreamlineProcessor originProcessor2 = TopologyTestHelper.createStreamlineProcessor("2");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originProcessor);
    originTopologyDag.add(originProcessor2);
    originTopologyDag.addEdge(new Edge("e1", originProcessor, originProcessor2, "default", Stream.Grouping.SHUFFLE));
    TestRunProcessor testProcessor = createTestRunProcessor(originProcessor);
    TestRunProcessor testProcessor2 = createTestRunProcessor(originProcessor2);
    Map<String, TestRunProcessor> processorMap = new HashMap<>();
    processorMap.put(originProcessor.getName(), testProcessor);
    processorMap.put(originProcessor2.getName(), testProcessor2);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.emptyMap(), processorMap, Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originProcessor);
    visitor.visit(originProcessor2);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testProcessors = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunProcessor)).collect(toList());
    Optional<OutputComponent> testRunProcessorOptional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor.getName())).findAny();
    Optional<OutputComponent> testRunProcessor2Optional = testProcessors.stream().filter(o -> o.getName().equals(originProcessor2.getName())).findAny();
    assertTrue(testRunProcessorOptional.isPresent());
    assertTrue(testRunProcessor2Optional.isPresent());
    TestRunProcessor testRunProcessor = (TestRunProcessor) testRunProcessorOptional.get();
    TestRunProcessor testRunProcessor2 = (TestRunProcessor) testRunProcessor2Optional.get();
    assertEquals(1, testTopologyDag.getEdgesFrom(testRunProcessor).size());
    assertEquals(1, testTopologyDag.getEdgesTo(testRunProcessor2).size());
    assertTrue(testTopologyDag.getEdgesFrom(testRunProcessor).get(0) == testTopologyDag.getEdgesTo(testRunProcessor2).get(0));
}

Example 10 with TopologyDag

use of com.hortonworks.streamline.streams.layout.component.TopologyDag in project streamline by hortonworks.

the class TestTopologyDagCreatingVisitorTest method visitSource.

@Test
public void visitSource() throws Exception {
    StreamlineSource originSource = TopologyTestHelper.createStreamlineSource("1");
    TopologyDag originTopologyDag = new TopologyDag();
    originTopologyDag.add(originSource);
    TestRunSource testSource = createTestRunSource(originSource);
    TestTopologyDagCreatingVisitor visitor = new TestTopologyDagCreatingVisitor(originTopologyDag, Collections.singletonMap(originSource.getName(), testSource), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    visitor.visit(originSource);
    TopologyDag testTopologyDag = visitor.getTestTopologyDag();
    List<OutputComponent> testSources = testTopologyDag.getOutputComponents().stream().filter(o -> (o instanceof TestRunSource && o.getName().equals(originSource.getName()))).collect(toList());
    assertEquals(1, testSources.size());
    TestRunSource testRunSource = (TestRunSource) testSources.get(0);
    assertEquals(originSource.getId(), testRunSource.getId());
    assertEquals(testSource.getTestRecordsForEachStream(), testRunSource.getTestRecordsForEachStream());
}