Examples with Config org.apache.heron.spi.common.Config used on opensource projects

Example 76 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class Runtime method main.

@SuppressWarnings({ "IllegalCatch", "RegexpSinglelineJava" })
public static void main(String[] args) throws Exception {
    final Options options = createOptions();
    final Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption(Flag.Help.name)) {
        usage(options);
        return;
    }
    try {
        cmd = parser.parse(options, args);
    } catch (ParseException pe) {
        System.err.println(pe.getMessage());
        usage(options);
        return;
    }
    final boolean verbose = isVerbose(cmd);
    // set and configure logging level
    Logging.setVerbose(verbose);
    Logging.configure(verbose);
    LOG.debug("API server overrides:\n {}", cmd.getOptionProperties(Flag.Property.name));
    final String toolsHome = getToolsHome();
    // read command line flags
    final String cluster = cmd.getOptionValue(Flag.Cluster.name);
    final String heronConfigurationDirectory = getConfigurationDirectory(toolsHome, cmd);
    final String heronDirectory = getHeronDirectory(cmd);
    final String releaseFile = getReleaseFile(toolsHome, cmd);
    final String configurationOverrides = loadOverrides(cmd);
    final int port = getPort(cmd);
    final String downloadHostName = getDownloadHostName(cmd);
    final String heronCorePackagePath = getHeronCorePackagePath(cmd);
    final Config baseConfiguration = ConfigUtils.getBaseConfiguration(heronDirectory, heronConfigurationDirectory, releaseFile, configurationOverrides);
    final ResourceConfig config = new ResourceConfig(Resources.get());
    final Server server = new Server(port);
    final ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
    contextHandler.setContextPath("/");
    LOG.info("using configuration path: {}", heronConfigurationDirectory);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_CLUSTER, cluster);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION, baseConfiguration);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_DIRECTORY, heronConfigurationDirectory);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH, configurationOverrides);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_PORT, String.valueOf(port));
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_DOWNLOAD_HOSTNAME, Utils.isNotEmpty(downloadHostName) ? String.valueOf(downloadHostName) : null);
    contextHandler.setAttribute(HeronResource.ATTRIBUTE_HERON_CORE_PACKAGE_PATH, Utils.isNotEmpty(heronCorePackagePath) ? String.valueOf(heronCorePackagePath) : null);
    server.setHandler(contextHandler);
    final ServletHolder apiServlet = new ServletHolder(new ServletContainer(config));
    contextHandler.addServlet(apiServlet, API_BASE_PATH);
    try {
        server.start();
        LOG.info("Heron API server started at {}", server.getURI());
        server.join();
    } catch (Exception ex) {
        final String message = getErrorMessage(server, port, ex);
        LOG.error(message);
        System.err.println(message);
        System.exit(1);
    } finally {
        server.destroy();
    }
}

Example 77 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class FileResource method uploadFile.

/**
 * Endpoints for artifacts upload
 */
@POST
@Path("/upload")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response uploadFile(@FormDataParam("file") InputStream uploadedInputStream, @FormDataParam("file") FormDataContentDisposition fileDetail) {
    Config config = createConfig();
    if (uploadedInputStream == null) {
        String msg = "input stream is null";
        LOG.error(msg);
        return Response.status(Response.Status.BAD_REQUEST).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(msg)).build();
    }
    if (fileDetail == null) {
        String msg = "form data content disposition is null";
        LOG.error(msg);
        return Response.status(Response.Status.BAD_REQUEST).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(msg)).build();
    }
    String uploadDir = config.getStringValue(FILE_SYSTEM_DIRECTORY);
    final String fileName = UUID.randomUUID() + "-" + fileDetail.getFileName();
    final String uploadedFileLocation = uploadDir + "/" + fileName;
    // save it
    try {
        FileHelper.writeToFile(uploadedInputStream, uploadedFileLocation);
    } catch (IOException e) {
        LOG.error("error uploading file {}", fileDetail.getFileName(), e);
        return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(e.getMessage())).build();
    }
    String uri = String.format("http://%s:%s/api/v1/file/download/%s", getHostNameOrIP(), getPort(), fileName);
    return Response.status(Response.Status.OK).entity(uri).build();
}

Example 78 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class FileResource method downloadFile.

/**
 * Endpoints for artifacts download
 */
@GET
@Path("/download/{file}")
public Response downloadFile(@PathParam("file") final String file) {
    Config config = createConfig();
    String uploadDir = config.getStringValue(FILE_SYSTEM_DIRECTORY);
    String filePath = uploadDir + "/" + file;
    return getResponseByFile(filePath);
}

Example 79 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class TopologyResource method submit.

@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings({ "IllegalCatch", "JavadocMethod" })
public Response submit(FormDataMultiPart form) throws IOException {
    // verify that all we have all the required params
    final List<String> missingDataKeys = verifyKeys(form.getFields().keySet(), REQUIRED_SUBMIT_TOPOLOGY_PARAMS);
    if (!missingDataKeys.isEmpty()) {
        // return error since we are missing required parameters
        final String message = String.format("Validation failed missing required params: %s", missingDataKeys.toString());
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createValidationError(message, missingDataKeys)).build();
    }
    final String cluster = Forms.getString(form, FORM_KEY_CLUSTER);
    if (!doesClusterMatch(cluster)) {
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("Unknown cluster %s expecting '%s'", cluster, getCluster()))).build();
    }
    final String topologyName = Forms.getString(form, FORM_KEY_NAME);
    final String role = Forms.getString(form, FORM_KEY_ROLE);
    final String environment = Forms.getString(form, FORM_KEY_ENVIRONMENT, Constants.DEFAULT_HERON_ENVIRONMENT);
    final String user = Forms.getString(form, FORM_KEY_USER, role);
    // submit overrides are passed key=value
    final Map<String, String> submitOverrides = getSubmitOverrides(form);
    final String topologyDirectory = Files.createTempDirectory(topologyName).toFile().getAbsolutePath();
    try {
        // upload the topology definition file to the topology directory
        final FormDataBodyPart definitionFilePart = form.getField(FORM_KEY_DEFINITION);
        final File topologyDefinitionFile = Forms.uploadFile(definitionFilePart, topologyDirectory);
        // upload the topology binary file to the topology directory
        final FormDataBodyPart topologyFilePart = form.getField(FORM_KEY_TOPOLOGY);
        final File topologyBinaryFile = Forms.uploadFile(topologyFilePart, topologyDirectory);
        final boolean isDryRun = form.getFields().containsKey(PARAM_DRY_RUN);
        final boolean isVerbose = form.getFields().containsKey("verbose");
        final boolean isVerboseGC = form.getFields().containsKey("verbose_gc");
        // copy configuration files to the sandbox config location
        // topology-dir/<default-heron-sandbox-config>
        FileHelper.copyDirectory(Paths.get(getConfigurationDirectory()), Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG));
        final java.nio.file.Path overridesPath = Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.OVERRIDE_FILE);
        // copy override file into topology configuration directory
        FileHelper.copy(Paths.get(getConfigurationOverridePath()), overridesPath);
        // apply submit overrides
        ConfigUtils.applyOverrides(overridesPath, submitOverrides);
        // apply overrides to state manager config
        ConfigUtils.applyOverridesToStateManagerConfig(overridesPath, Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.STATE_MANAGER_FILE));
        // create tar file from the contents of the topology directory
        final File topologyPackageFile = Paths.get(topologyDirectory, TOPOLOGY_TAR_GZ_FILENAME).toFile();
        FileHelper.createTarGz(topologyPackageFile, FileHelper.getChildren(topologyDirectory));
        // create configs
        Config topologyConfig = ConfigUtils.getTopologyConfig(topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath());
        List<Pair<String, Object>> val = new LinkedList<>();
        for (Map.Entry<String, Object> entry : topologyConfig.getEntrySet()) {
            val.add(Pair.create(entry.getKey(), entry.getValue()));
        }
        val.addAll(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.TOPOLOGY_NAME.value(), topologyName), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.SUBMIT_USER.value(), user), Pair.create(Key.DRY_RUN.value(), isDryRun), Pair.create(Key.VERBOSE.value(), isVerbose), Pair.create(Key.VERBOSE_GC.value(), isVerboseGC)));
        final Config config = createConfig(val, submitOverrides);
        // submit the topology
        getActionFactory().createSubmitAction(config, topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath()).execute();
        return Response.created(URI.create(String.format(TOPOLOGY_PATH_FORMAT, cluster, role, environment, topologyName))).type(MediaType.APPLICATION_JSON).entity(createdResponse(cluster, role, environment, topologyName)).build();
    } catch (SubmitDryRunResponse response) {
        return createDryRunResponse(response, Forms.getString(form, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
    } catch (Exception ex) {
        LOG.error("error submitting topology {}", topologyName, ex);
        return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(ex.getMessage())).build();
    } finally {
        FileUtils.deleteDir(topologyDirectory);
    }
}

Example 80 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class PackingTestUtils method testPackingPlan.

public static PackingPlan testPackingPlan(String topologyName, IPacking packing) {
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("testSpout", 2);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("testBolt", 3);
    org.apache.heron.api.Config topologyConfig = new org.apache.heron.api.Config();
    topologyConfig.put(org.apache.heron.api.Config.TOPOLOGY_STMGRS, 1);
    TopologyAPI.Topology topology = TopologyTests.createTopology(topologyName, topologyConfig, spouts, bolts);
    Config config = Config.newBuilder(true).put(Key.TOPOLOGY_ID, topology.getId()).put(Key.TOPOLOGY_NAME, topology.getName()).build();
    packing.initialize(config, topology);
    return packing.pack();
}