use of org.apache.heron.spi.common.Config in project heron by twitter.
the class Runtime method main.
@SuppressWarnings({ "IllegalCatch", "RegexpSinglelineJava" })
public static void main(String[] args) throws Exception {
final Options options = createOptions();
final Options helpOptions = constructHelpOptions();
CommandLineParser parser = new DefaultParser();
// parse the help options first.
CommandLine cmd = parser.parse(helpOptions, args, true);
if (cmd.hasOption(Flag.Help.name)) {
usage(options);
return;
}
try {
cmd = parser.parse(options, args);
} catch (ParseException pe) {
System.err.println(pe.getMessage());
usage(options);
return;
}
final boolean verbose = isVerbose(cmd);
// set and configure logging level
Logging.setVerbose(verbose);
Logging.configure(verbose);
LOG.debug("API server overrides:\n {}", cmd.getOptionProperties(Flag.Property.name));
final String toolsHome = getToolsHome();
// read command line flags
final String cluster = cmd.getOptionValue(Flag.Cluster.name);
final String heronConfigurationDirectory = getConfigurationDirectory(toolsHome, cmd);
final String heronDirectory = getHeronDirectory(cmd);
final String releaseFile = getReleaseFile(toolsHome, cmd);
final String configurationOverrides = loadOverrides(cmd);
final int port = getPort(cmd);
final String downloadHostName = getDownloadHostName(cmd);
final String heronCorePackagePath = getHeronCorePackagePath(cmd);
final Config baseConfiguration = ConfigUtils.getBaseConfiguration(heronDirectory, heronConfigurationDirectory, releaseFile, configurationOverrides);
final ResourceConfig config = new ResourceConfig(Resources.get());
final Server server = new Server(port);
final ServletContextHandler contextHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
contextHandler.setContextPath("/");
LOG.info("using configuration path: {}", heronConfigurationDirectory);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_CLUSTER, cluster);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION, baseConfiguration);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_DIRECTORY, heronConfigurationDirectory);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_CONFIGURATION_OVERRIDE_PATH, configurationOverrides);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_PORT, String.valueOf(port));
contextHandler.setAttribute(HeronResource.ATTRIBUTE_DOWNLOAD_HOSTNAME, Utils.isNotEmpty(downloadHostName) ? String.valueOf(downloadHostName) : null);
contextHandler.setAttribute(HeronResource.ATTRIBUTE_HERON_CORE_PACKAGE_PATH, Utils.isNotEmpty(heronCorePackagePath) ? String.valueOf(heronCorePackagePath) : null);
server.setHandler(contextHandler);
final ServletHolder apiServlet = new ServletHolder(new ServletContainer(config));
contextHandler.addServlet(apiServlet, API_BASE_PATH);
try {
server.start();
LOG.info("Heron API server started at {}", server.getURI());
server.join();
} catch (Exception ex) {
final String message = getErrorMessage(server, port, ex);
LOG.error(message);
System.err.println(message);
System.exit(1);
} finally {
server.destroy();
}
}
use of org.apache.heron.spi.common.Config in project heron by twitter.
the class FileResource method uploadFile.
/**
* Endpoints for artifacts upload
*/
@POST
@Path("/upload")
@Consumes(MediaType.MULTIPART_FORM_DATA)
public Response uploadFile(@FormDataParam("file") InputStream uploadedInputStream, @FormDataParam("file") FormDataContentDisposition fileDetail) {
Config config = createConfig();
if (uploadedInputStream == null) {
String msg = "input stream is null";
LOG.error(msg);
return Response.status(Response.Status.BAD_REQUEST).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(msg)).build();
}
if (fileDetail == null) {
String msg = "form data content disposition is null";
LOG.error(msg);
return Response.status(Response.Status.BAD_REQUEST).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(msg)).build();
}
String uploadDir = config.getStringValue(FILE_SYSTEM_DIRECTORY);
final String fileName = UUID.randomUUID() + "-" + fileDetail.getFileName();
final String uploadedFileLocation = uploadDir + "/" + fileName;
// save it
try {
FileHelper.writeToFile(uploadedInputStream, uploadedFileLocation);
} catch (IOException e) {
LOG.error("error uploading file {}", fileDetail.getFileName(), e);
return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(e.getMessage())).build();
}
String uri = String.format("http://%s:%s/api/v1/file/download/%s", getHostNameOrIP(), getPort(), fileName);
return Response.status(Response.Status.OK).entity(uri).build();
}
use of org.apache.heron.spi.common.Config in project heron by twitter.
the class FileResource method downloadFile.
/**
* Endpoints for artifacts download
*/
@GET
@Path("/download/{file}")
public Response downloadFile(@PathParam("file") final String file) {
Config config = createConfig();
String uploadDir = config.getStringValue(FILE_SYSTEM_DIRECTORY);
String filePath = uploadDir + "/" + file;
return getResponseByFile(filePath);
}
use of org.apache.heron.spi.common.Config in project heron by twitter.
the class TopologyResource method submit.
@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings({ "IllegalCatch", "JavadocMethod" })
public Response submit(FormDataMultiPart form) throws IOException {
// verify that all we have all the required params
final List<String> missingDataKeys = verifyKeys(form.getFields().keySet(), REQUIRED_SUBMIT_TOPOLOGY_PARAMS);
if (!missingDataKeys.isEmpty()) {
// return error since we are missing required parameters
final String message = String.format("Validation failed missing required params: %s", missingDataKeys.toString());
return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createValidationError(message, missingDataKeys)).build();
}
final String cluster = Forms.getString(form, FORM_KEY_CLUSTER);
if (!doesClusterMatch(cluster)) {
return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("Unknown cluster %s expecting '%s'", cluster, getCluster()))).build();
}
final String topologyName = Forms.getString(form, FORM_KEY_NAME);
final String role = Forms.getString(form, FORM_KEY_ROLE);
final String environment = Forms.getString(form, FORM_KEY_ENVIRONMENT, Constants.DEFAULT_HERON_ENVIRONMENT);
final String user = Forms.getString(form, FORM_KEY_USER, role);
// submit overrides are passed key=value
final Map<String, String> submitOverrides = getSubmitOverrides(form);
final String topologyDirectory = Files.createTempDirectory(topologyName).toFile().getAbsolutePath();
try {
// upload the topology definition file to the topology directory
final FormDataBodyPart definitionFilePart = form.getField(FORM_KEY_DEFINITION);
final File topologyDefinitionFile = Forms.uploadFile(definitionFilePart, topologyDirectory);
// upload the topology binary file to the topology directory
final FormDataBodyPart topologyFilePart = form.getField(FORM_KEY_TOPOLOGY);
final File topologyBinaryFile = Forms.uploadFile(topologyFilePart, topologyDirectory);
final boolean isDryRun = form.getFields().containsKey(PARAM_DRY_RUN);
final boolean isVerbose = form.getFields().containsKey("verbose");
final boolean isVerboseGC = form.getFields().containsKey("verbose_gc");
// copy configuration files to the sandbox config location
// topology-dir/<default-heron-sandbox-config>
FileHelper.copyDirectory(Paths.get(getConfigurationDirectory()), Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG));
final java.nio.file.Path overridesPath = Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.OVERRIDE_FILE);
// copy override file into topology configuration directory
FileHelper.copy(Paths.get(getConfigurationOverridePath()), overridesPath);
// apply submit overrides
ConfigUtils.applyOverrides(overridesPath, submitOverrides);
// apply overrides to state manager config
ConfigUtils.applyOverridesToStateManagerConfig(overridesPath, Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.STATE_MANAGER_FILE));
// create tar file from the contents of the topology directory
final File topologyPackageFile = Paths.get(topologyDirectory, TOPOLOGY_TAR_GZ_FILENAME).toFile();
FileHelper.createTarGz(topologyPackageFile, FileHelper.getChildren(topologyDirectory));
// create configs
Config topologyConfig = ConfigUtils.getTopologyConfig(topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath());
List<Pair<String, Object>> val = new LinkedList<>();
for (Map.Entry<String, Object> entry : topologyConfig.getEntrySet()) {
val.add(Pair.create(entry.getKey(), entry.getValue()));
}
val.addAll(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.TOPOLOGY_NAME.value(), topologyName), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.SUBMIT_USER.value(), user), Pair.create(Key.DRY_RUN.value(), isDryRun), Pair.create(Key.VERBOSE.value(), isVerbose), Pair.create(Key.VERBOSE_GC.value(), isVerboseGC)));
final Config config = createConfig(val, submitOverrides);
// submit the topology
getActionFactory().createSubmitAction(config, topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath()).execute();
return Response.created(URI.create(String.format(TOPOLOGY_PATH_FORMAT, cluster, role, environment, topologyName))).type(MediaType.APPLICATION_JSON).entity(createdResponse(cluster, role, environment, topologyName)).build();
} catch (SubmitDryRunResponse response) {
return createDryRunResponse(response, Forms.getString(form, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
} catch (Exception ex) {
LOG.error("error submitting topology {}", topologyName, ex);
return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(ex.getMessage())).build();
} finally {
FileUtils.deleteDir(topologyDirectory);
}
}
use of org.apache.heron.spi.common.Config in project heron by twitter.
the class PackingTestUtils method testPackingPlan.
public static PackingPlan testPackingPlan(String topologyName, IPacking packing) {
Map<String, Integer> spouts = new HashMap<>();
spouts.put("testSpout", 2);
Map<String, Integer> bolts = new HashMap<>();
bolts.put("testBolt", 3);
org.apache.heron.api.Config topologyConfig = new org.apache.heron.api.Config();
topologyConfig.put(org.apache.heron.api.Config.TOPOLOGY_STMGRS, 1);
TopologyAPI.Topology topology = TopologyTests.createTopology(topologyName, topologyConfig, spouts, bolts);
Config config = Config.newBuilder(true).put(Key.TOPOLOGY_ID, topology.getId()).put(Key.TOPOLOGY_NAME, topology.getName()).build();
packing.initialize(config, topology);
return packing.pack();
}
Aggregations