use of org.apache.heron.spi.packing.PackingException in project heron by twitter.
the class ResourceCompliantRRPacking method retryWithAdditionalContainer.
private void retryWithAdditionalContainer() {
increaseNumContainers(1);
resetToFirstContainer();
int totalInstances = TopologyUtils.getTotalInstance(topology);
if (numContainers > totalInstances) {
throw new PackingException("Cannot add to that container");
}
}
use of org.apache.heron.spi.packing.PackingException in project heron by twitter.
the class SubmitterMainTest method testSubmitTopologyLauncherException.
@Test(expected = PackingException.class)
public void testSubmitTopologyLauncherException() throws Exception {
SubmitterMain submitterMain = spy(new SubmitterMain(config, topology));
doNothing().when(submitterMain).validateSubmit(any(SchedulerStateManagerAdaptor.class), anyString());
final URI packageURI = new URI("mock://uri:924/x#ke");
doReturn(packageURI).when(submitterMain).uploadPackage(eq(uploader));
doThrow(new PackingException("")).when(submitterMain).callLauncherRunner(Mockito.any(Config.class));
submitterMain.submitTopology();
}
use of org.apache.heron.spi.packing.PackingException in project heron by twitter.
the class SubmitterMain method submitTopology.
/**
* Submit a topology
* 1. Instantiate necessary resources
* 2. Valid whether it is legal to submit a topology
* 3. Call LauncherRunner
*/
public void submitTopology() throws TopologySubmissionException {
// build primary runtime config first
Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
// call launcher directly here if in dry-run mode
if (Context.dryRun(config)) {
callLauncherRunner(primaryRuntime);
return;
}
// 1. Do prepare work
// create an instance of state manager
String statemgrClass = Context.stateManagerClass(config);
IStateManager statemgr;
// Create an instance of the launcher class
String launcherClass = Context.launcherClass(config);
ILauncher launcher;
// create an instance of the uploader class
String uploaderClass = Context.uploaderClass(config);
IUploader uploader;
// create an instance of state manager
try {
statemgr = ReflectionUtils.newInstance(statemgrClass);
} catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
}
// create an instance of launcher
try {
launcher = ReflectionUtils.newInstance(launcherClass);
} catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
}
// create an instance of uploader
try {
uploader = ReflectionUtils.newInstance(uploaderClass);
} catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
}
// Put it in a try block so that we can always clean resources
try {
// initialize the state manager
statemgr.initialize(config);
// initialize the uploader
uploader.initialize(config);
// TODO(mfu): timeout should read from config
SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
// Check if topology is already running
validateSubmit(adaptor, topology.getName());
LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
// The packing plan might call for a number of containers different than the config
// settings. If that's the case we need to modify the configs to match.
runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
// If the packing plan is valid we will upload necessary packages
URI packageURI = uploadPackage(uploader);
// Update the runtime config with the packageURI
Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
callLauncherRunner(runtimeAll);
} catch (LauncherException | PackingException e) {
// we undo uploading of topology package only if launcher fails to
// launch topology, which will throw LauncherException or PackingException
uploader.undo();
throw e;
} finally {
SysUtils.closeIgnoringExceptions(uploader);
SysUtils.closeIgnoringExceptions(launcher);
SysUtils.closeIgnoringExceptions(statemgr);
}
}
use of org.apache.heron.spi.packing.PackingException in project heron by twitter.
the class RoundRobinPacking method calculateInstancesResourceMapInContainer.
@SuppressWarnings("unchecked")
private <T extends ResourceMeasure> Map<Integer, Map<InstanceId, T>> calculateInstancesResourceMapInContainer(Map<Integer, List<InstanceId>> allocation, Map<String, T> resMap, T containerResHint, T defaultContainerRes, T instanceResDefault, T containerResPadding, T zero, T notSpecified, String resourceType) {
Map<Integer, Map<InstanceId, T>> instancesResMapInContainer = new HashMap<>();
for (int containerId : allocation.keySet()) {
List<InstanceId> instanceIds = allocation.get(containerId);
Map<InstanceId, T> resInsideContainer = new HashMap<>();
instancesResMapInContainer.put(containerId, resInsideContainer);
List<InstanceId> unspecifiedInstances = new ArrayList<>();
// Register the instance resource allocation and calculate the used resource so far
T usedRes = zero;
for (InstanceId instanceId : instanceIds) {
String componentName = instanceId.getComponentName();
if (resMap.containsKey(componentName)) {
T res = resMap.get(componentName);
resInsideContainer.put(instanceId, res);
usedRes = (T) usedRes.plus(res);
} else {
unspecifiedInstances.add(instanceId);
}
}
// Soft padding constraint validation: warn if padding amount cannot be accommodated
boolean paddingThrottling = false;
if (!containerResHint.equals(notSpecified) && usedRes.greaterThan(containerResHint.minus(containerResPadding))) {
// Validate instance resources specified so far don't violate container-level constraint
if (usedRes.greaterThan(containerResHint)) {
throw new PackingException(String.format("Invalid packing plan generated. " + "Total instance %s (%s) in container#%d have exceeded " + "the container-level constraint of %s.", resourceType, usedRes.toString(), containerId, containerResHint.toString()));
}
paddingThrottling = true;
LOG.warning(String.format("Container#%d (max %s: %s) is now hosting instances that " + "take up to %s %s. The container may not have enough resource to accommodate " + "internal processes which take up to %s %s.", containerId, resourceType, containerResHint.toString(), usedRes.toString(), resourceType, containerResPadding.toString(), resourceType));
}
// calculate resource for the remaining unspecified instances if any
T containerRes = containerResHint;
if (containerResHint.equals(notSpecified)) {
containerRes = defaultContainerRes;
}
if (!unspecifiedInstances.isEmpty()) {
T individualInstanceRes = instanceResDefault;
// discount resource for heron internal process (padding) and used (usedRes)
T remainingRes;
if (paddingThrottling) {
remainingRes = (T) containerRes.minus(usedRes);
} else {
remainingRes = (T) containerRes.minus(containerResPadding).minus(usedRes);
}
if (remainingRes.lessOrEqual(zero)) {
throw new PackingException(String.format("Invalid packing plan generated. " + "No enough %s to allocate for unspecified instances", resourceType));
}
// Split remaining resource evenly
individualInstanceRes = (T) remainingRes.divide(unspecifiedInstances.size());
// Put the results in resInsideContainer
for (InstanceId instanceId : unspecifiedInstances) {
resInsideContainer.put(instanceId, individualInstanceRes);
}
}
}
return instancesResMapInContainer;
}
use of org.apache.heron.spi.packing.PackingException in project heron by twitter.
the class LaunchRunner method call.
/**
* Call launcher to launch topology
*
* @throws LauncherException
* @throws PackingException
* @throws SubmitDryRunResponse
*/
public void call() throws LauncherException, PackingException, SubmitDryRunResponse {
SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
TopologyAPI.Topology topology = Runtime.topology(runtime);
String topologyName = Context.topologyName(config);
PackingPlan packedPlan = LauncherUtils.getInstance().createPackingPlan(config, runtime);
if (Context.dryRun(config)) {
throw new SubmitDryRunResponse(topology, config, packedPlan);
}
// initialize the launcher
launcher.initialize(config, runtime);
// Set topology def first since we determine whether a topology is running
// by checking the existence of topology def
// store the trimmed topology definition into the state manager
// TODO(rli): log-and-false anti-pattern is too nested on this path. will not refactor
Boolean result = statemgr.setTopology(trimTopology(topology), topologyName);
if (result == null || !result) {
throw new LauncherException(String.format("Failed to set topology definition for topology '%s'", topologyName));
}
result = statemgr.setPackingPlan(createPackingPlan(packedPlan), topologyName);
if (result == null || !result) {
statemgr.deleteTopology(topologyName);
throw new LauncherException(String.format("Failed to set packing plan for topology '%s'", topologyName));
}
// store the execution state into the state manager
ExecutionEnvironment.ExecutionState executionState = createExecutionState();
result = statemgr.setExecutionState(executionState, topologyName);
if (result == null || !result) {
statemgr.deletePackingPlan(topologyName);
statemgr.deleteTopology(topologyName);
throw new LauncherException(String.format("Failed to set execution state for topology '%s'", topologyName));
}
// returning false. In some cases the scheduler needs to have the topology deleted.
try {
if (!launcher.launch(packedPlan)) {
throw new TopologySubmissionException(null);
}
} catch (TopologySubmissionException e) {
// Compile error message to throw.
final StringBuilder errorMessage = new StringBuilder(String.format("Failed to launch topology '%s'", topologyName));
if (e.getMessage() != null) {
errorMessage.append("\n").append(e.getMessage());
}
try {
// Clear state from the Scheduler via RPC.
Scheduler.KillTopologyRequest killTopologyRequest = Scheduler.KillTopologyRequest.newBuilder().setTopologyName(topologyName).build();
ISchedulerClient schedulerClient = new SchedulerClientFactory(config, runtime).getSchedulerClient();
if (!schedulerClient.killTopology(killTopologyRequest)) {
final String logMessage = String.format("Failed to remove topology '%s' from scheduler after failed submit. " + "Please re-try the kill command.", topologyName);
errorMessage.append("\n").append(logMessage);
LOG.log(Level.SEVERE, logMessage);
}
// SUPPRESS CHECKSTYLE IllegalCatch
} catch (Exception ignored) {
// The above call to clear the Scheduler may fail. This situation can be ignored.
LOG.log(Level.FINE, String.format("Failure clearing failed topology `%s` from Scheduler during `submit`", topologyName));
}
// Clear state from the State Manager.
statemgr.deleteExecutionState(topologyName);
statemgr.deletePackingPlan(topologyName);
statemgr.deleteTopology(topologyName);
throw new LauncherException(errorMessage.toString());
}
}
Aggregations