Examples with SubmitDryRunResponse org.apache.heron.scheduler.dryrun.SubmitDryRunResponse used on opensource projects

Example 1 with SubmitDryRunResponse

use of org.apache.heron.scheduler.dryrun.SubmitDryRunResponse in project heron by twitter.

the class TopologyResource method submit.

@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings({ "IllegalCatch", "JavadocMethod" })
public Response submit(FormDataMultiPart form) throws IOException {
    // verify that all we have all the required params
    final List<String> missingDataKeys = verifyKeys(form.getFields().keySet(), REQUIRED_SUBMIT_TOPOLOGY_PARAMS);
    if (!missingDataKeys.isEmpty()) {
        // return error since we are missing required parameters
        final String message = String.format("Validation failed missing required params: %s", missingDataKeys.toString());
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createValidationError(message, missingDataKeys)).build();
    }
    final String cluster = Forms.getString(form, FORM_KEY_CLUSTER);
    if (!doesClusterMatch(cluster)) {
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("Unknown cluster %s expecting '%s'", cluster, getCluster()))).build();
    }
    final String topologyName = Forms.getString(form, FORM_KEY_NAME);
    final String role = Forms.getString(form, FORM_KEY_ROLE);
    final String environment = Forms.getString(form, FORM_KEY_ENVIRONMENT, Constants.DEFAULT_HERON_ENVIRONMENT);
    final String user = Forms.getString(form, FORM_KEY_USER, role);
    // submit overrides are passed key=value
    final Map<String, String> submitOverrides = getSubmitOverrides(form);
    final String topologyDirectory = Files.createTempDirectory(topologyName).toFile().getAbsolutePath();
    try {
        // upload the topology definition file to the topology directory
        final FormDataBodyPart definitionFilePart = form.getField(FORM_KEY_DEFINITION);
        final File topologyDefinitionFile = Forms.uploadFile(definitionFilePart, topologyDirectory);
        // upload the topology binary file to the topology directory
        final FormDataBodyPart topologyFilePart = form.getField(FORM_KEY_TOPOLOGY);
        final File topologyBinaryFile = Forms.uploadFile(topologyFilePart, topologyDirectory);
        final boolean isDryRun = form.getFields().containsKey(PARAM_DRY_RUN);
        final boolean isVerbose = form.getFields().containsKey("verbose");
        final boolean isVerboseGC = form.getFields().containsKey("verbose_gc");
        // copy configuration files to the sandbox config location
        // topology-dir/<default-heron-sandbox-config>
        FileHelper.copyDirectory(Paths.get(getConfigurationDirectory()), Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG));
        final java.nio.file.Path overridesPath = Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.OVERRIDE_FILE);
        // copy override file into topology configuration directory
        FileHelper.copy(Paths.get(getConfigurationOverridePath()), overridesPath);
        // apply submit overrides
        ConfigUtils.applyOverrides(overridesPath, submitOverrides);
        // apply overrides to state manager config
        ConfigUtils.applyOverridesToStateManagerConfig(overridesPath, Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.STATE_MANAGER_FILE));
        // create tar file from the contents of the topology directory
        final File topologyPackageFile = Paths.get(topologyDirectory, TOPOLOGY_TAR_GZ_FILENAME).toFile();
        FileHelper.createTarGz(topologyPackageFile, FileHelper.getChildren(topologyDirectory));
        // create configs
        Config topologyConfig = ConfigUtils.getTopologyConfig(topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath());
        List<Pair<String, Object>> val = new LinkedList<>();
        for (Map.Entry<String, Object> entry : topologyConfig.getEntrySet()) {
            val.add(Pair.create(entry.getKey(), entry.getValue()));
        }
        val.addAll(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.TOPOLOGY_NAME.value(), topologyName), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.SUBMIT_USER.value(), user), Pair.create(Key.DRY_RUN.value(), isDryRun), Pair.create(Key.VERBOSE.value(), isVerbose), Pair.create(Key.VERBOSE_GC.value(), isVerboseGC)));
        final Config config = createConfig(val, submitOverrides);
        // submit the topology
        getActionFactory().createSubmitAction(config, topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath()).execute();
        return Response.created(URI.create(String.format(TOPOLOGY_PATH_FORMAT, cluster, role, environment, topologyName))).type(MediaType.APPLICATION_JSON).entity(createdResponse(cluster, role, environment, topologyName)).build();
    } catch (SubmitDryRunResponse response) {
        return createDryRunResponse(response, Forms.getString(form, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
    } catch (Exception ex) {
        LOG.error("error submitting topology {}", topologyName, ex);
        return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(ex.getMessage())).build();
    } finally {
        FileUtils.deleteDir(topologyDirectory);
    }
}

Example 2 with SubmitDryRunResponse

use of org.apache.heron.scheduler.dryrun.SubmitDryRunResponse in project heron by twitter.

the class LaunchRunner method call.

/**
 * Call launcher to launch topology
 *
 * @throws LauncherException
 * @throws PackingException
 * @throws SubmitDryRunResponse
 */
public void call() throws LauncherException, PackingException, SubmitDryRunResponse {
    SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
    TopologyAPI.Topology topology = Runtime.topology(runtime);
    String topologyName = Context.topologyName(config);
    PackingPlan packedPlan = LauncherUtils.getInstance().createPackingPlan(config, runtime);
    if (Context.dryRun(config)) {
        throw new SubmitDryRunResponse(topology, config, packedPlan);
    }
    // initialize the launcher
    launcher.initialize(config, runtime);
    // Set topology def first since we determine whether a topology is running
    // by checking the existence of topology def
    // store the trimmed topology definition into the state manager
    // TODO(rli): log-and-false anti-pattern is too nested on this path. will not refactor
    Boolean result = statemgr.setTopology(trimTopology(topology), topologyName);
    if (result == null || !result) {
        throw new LauncherException(String.format("Failed to set topology definition for topology '%s'", topologyName));
    }
    result = statemgr.setPackingPlan(createPackingPlan(packedPlan), topologyName);
    if (result == null || !result) {
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set packing plan for topology '%s'", topologyName));
    }
    // store the execution state into the state manager
    ExecutionEnvironment.ExecutionState executionState = createExecutionState();
    result = statemgr.setExecutionState(executionState, topologyName);
    if (result == null || !result) {
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(String.format("Failed to set execution state for topology '%s'", topologyName));
    }
    // returning false. In some cases the scheduler needs to have the topology deleted.
    try {
        if (!launcher.launch(packedPlan)) {
            throw new TopologySubmissionException(null);
        }
    } catch (TopologySubmissionException e) {
        // Compile error message to throw.
        final StringBuilder errorMessage = new StringBuilder(String.format("Failed to launch topology '%s'", topologyName));
        if (e.getMessage() != null) {
            errorMessage.append("\n").append(e.getMessage());
        }
        try {
            // Clear state from the Scheduler via RPC.
            Scheduler.KillTopologyRequest killTopologyRequest = Scheduler.KillTopologyRequest.newBuilder().setTopologyName(topologyName).build();
            ISchedulerClient schedulerClient = new SchedulerClientFactory(config, runtime).getSchedulerClient();
            if (!schedulerClient.killTopology(killTopologyRequest)) {
                final String logMessage = String.format("Failed to remove topology '%s' from scheduler after failed submit. " + "Please re-try the kill command.", topologyName);
                errorMessage.append("\n").append(logMessage);
                LOG.log(Level.SEVERE, logMessage);
            }
        // SUPPRESS CHECKSTYLE IllegalCatch
        } catch (Exception ignored) {
            // The above call to clear the Scheduler may fail. This situation can be ignored.
            LOG.log(Level.FINE, String.format("Failure clearing failed topology `%s` from Scheduler during `submit`", topologyName));
        }
        // Clear state from the State Manager.
        statemgr.deleteExecutionState(topologyName);
        statemgr.deletePackingPlan(topologyName);
        statemgr.deleteTopology(topologyName);
        throw new LauncherException(errorMessage.toString());
    }
}

Example 3 with SubmitDryRunResponse

use of org.apache.heron.scheduler.dryrun.SubmitDryRunResponse in project heron by twitter.

the class SubmitterMain method main.

public static void main(String[] args) throws Exception {
    Options options = constructOptions();
    Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        usage(options);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    Level logLevel = Level.INFO;
    if (isVerbose(cmd)) {
        logLevel = Level.ALL;
    }
    // init log
    LoggingHelper.loggerInit(logLevel, false);
    // load the topology definition into topology proto
    TopologyAPI.Topology topology = TopologyUtils.getTopology(cmd.getOptionValue("topology_defn"));
    Config config = loadConfig(cmd, topology);
    LOG.fine("Static config loaded successfully");
    LOG.fine(config.toString());
    SubmitterMain submitterMain = new SubmitterMain(config, topology);
    /* Meaning of exit status code:
       - status code = 0:
         program exits without error
       - 0 < status code < 100:
         program fails to execute before program execution. For example,
         JVM cannot find or load main class
       - 100 <= status code < 200:
         program fails to launch after program execution. For example,
         topology definition file fails to be loaded
       - status code >= 200
         program sends out dry-run response */
    try {
        submitterMain.submitTopology();
    } catch (SubmitDryRunResponse response) {
        LOG.log(Level.FINE, "Sending out dry-run response");
        // Output may contain UTF-8 characters, so we should print using UTF-8 encoding
        PrintStream out = new PrintStream(System.out, true, StandardCharsets.UTF_8.name());
        out.print(DryRunRenders.render(response, Context.dryRunFormatType(config)));
        // Exit with status code 200 to indicate dry-run response is sent out
        // SUPPRESS CHECKSTYLE RegexpSinglelineJava
        System.exit(200);
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (Exception e) {
        /* Since only stderr is used (by logging), we use stdout here to
         propagate error message back to Python's executor.py (invoke site). */
        LOG.log(Level.SEVERE, "Exception when submitting topology", e);
        System.out.println(e.getMessage());
        // Exit with status code 100 to indicate that error has happened on user-land
        // SUPPRESS CHECKSTYLE RegexpSinglelineJava
        System.exit(100);
    }
    LOG.log(Level.FINE, "Topology {0} submitted successfully", topology.getName());
}