Examples with PackingException com.twitter.heron.spi.packing.PackingException used on opensource projects

Example 6 with PackingException

use of com.twitter.heron.spi.packing.PackingException in project heron by twitter.

the class PackingPlanBuilder method initContainers.

private void initContainers() {
    assertResourceSettings();
    Map<Integer, Container> newContainerMap = this.containers;
    HashMap<String, TreeSet<Integer>> newComponentIndexes = this.componentIndexes;
    TreeSet<Integer> newTaskIds = this.taskIds;
    if (newComponentIndexes == null) {
        newComponentIndexes = new HashMap<>();
    }
    if (newTaskIds == null) {
        newTaskIds = new TreeSet<>();
    }
    // if this is the first time called, initialize container map with empty or existing containers
    if (newContainerMap == null) {
        if (this.existingPacking == null) {
            newContainerMap = new HashMap<>();
            for (int containerId = 1; containerId <= numContainers; containerId++) {
                newContainerMap.put(containerId, new Container(containerId, this.maxContainerResource, this.requestedContainerPadding));
            }
        } else {
            try {
                newContainerMap = getContainers(this.existingPacking, this.requestedContainerPadding, newComponentIndexes, newTaskIds);
            } catch (ResourceExceededException e) {
                throw new PackingException("Could not initialize containers using existing packing plan", e);
            }
        }
    }
    if (this.numContainers > newContainerMap.size()) {
        List<Scorer<Container>> scorers = new ArrayList<>();
        scorers.add(new ContainerIdScorer());
        List<Container> sortedContainers = sortContainers(scorers, newContainerMap.values());
        int nextContainerId = sortedContainers.get(sortedContainers.size() - 1).getContainerId() + 1;
        Resource capacity = newContainerMap.get(sortedContainers.get(0).getContainerId()).getCapacity();
        for (int i = 0; i < numContainers - newContainerMap.size(); i++) {
            newContainerMap.put(nextContainerId, new Container(nextContainerId, capacity, this.requestedContainerPadding));
            nextContainerId++;
        }
    }
    this.containers = newContainerMap;
    this.componentIndexes = newComponentIndexes;
    this.taskIds = newTaskIds;
}

Example 7 with PackingException

use of com.twitter.heron.spi.packing.PackingException in project heron by twitter.

the class PackingUtils method assertIsValidInstance.

/**
   * Verifies the Instance has enough RAM and that it can fit within the container limits.
   *
   * @param instanceResources The resources allocated to the instance
   * @throws PackingException if the instance is invalid
   */
private static void assertIsValidInstance(Resource instanceResources, ByteAmount minInstanceRam, Resource maxContainerResources, int paddingPercentage) throws PackingException {
    if (instanceResources.getRam().lessThan(minInstanceRam)) {
        throw new PackingException(String.format("Instance requires ram %s which is less than the minimum ram per instance of %s", instanceResources.getRam(), minInstanceRam));
    }
    ByteAmount instanceRam = instanceResources.getRam().increaseBy(paddingPercentage);
    if (instanceRam.greaterThan(maxContainerResources.getRam())) {
        throw new PackingException(String.format("This instance requires containers of at least %s ram. The current max container " + "size is %s", instanceRam, maxContainerResources.getRam()));
    }
    double instanceCpu = Math.round(PackingUtils.increaseBy(instanceResources.getCpu(), paddingPercentage));
    if (instanceCpu > maxContainerResources.getCpu()) {
        throw new PackingException(String.format("This instance requires containers with at least %s cpu cores. The current max container" + "size is %s cores", instanceCpu > maxContainerResources.getCpu(), maxContainerResources.getCpu()));
    }
    ByteAmount instanceDisk = instanceResources.getDisk().increaseBy(paddingPercentage);
    if (instanceDisk.greaterThan(maxContainerResources.getDisk())) {
        throw new PackingException(String.format("This instance requires containers of at least %s disk. The current max container" + "size is %s", instanceDisk, maxContainerResources.getDisk()));
    }
}

Example 8 with PackingException

use of com.twitter.heron.spi.packing.PackingException in project heron by twitter.

the class SubmitterMain method submitTopology.

/**
   * Submit a topology
   * 1. Instantiate necessary resources
   * 2. Valid whether it is legal to submit a topology
   * 3. Call LauncherRunner
   *
   */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 9 with PackingException

use of com.twitter.heron.spi.packing.PackingException in project heron by twitter.

the class SubmitterMainTest method testSubmitTopologyLauncherException.

@Test(expected = PackingException.class)
public void testSubmitTopologyLauncherException() throws Exception {
    SubmitterMain submitterMain = spy(new SubmitterMain(config, topology));
    doNothing().when(submitterMain).validateSubmit(any(SchedulerStateManagerAdaptor.class), anyString());
    final URI packageURI = new URI("mock://uri:924/x#ke");
    doReturn(packageURI).when(submitterMain).uploadPackage(eq(uploader));
    doThrow(new PackingException("")).when(submitterMain).callLauncherRunner(Mockito.any(Config.class));
    submitterMain.submitTopology();
}

Example 10 with PackingException

use of com.twitter.heron.spi.packing.PackingException in project incubator-heron by apache.

the class SubmitterMainTest method testSubmitTopologyLauncherException.

@Test(expected = PackingException.class)
public void testSubmitTopologyLauncherException() throws Exception {
    SubmitterMain submitterMain = spy(new SubmitterMain(config, topology));
    doNothing().when(submitterMain).validateSubmit(any(SchedulerStateManagerAdaptor.class), anyString());
    final URI packageURI = new URI("mock://uri:924/x#ke");
    doReturn(packageURI).when(submitterMain).uploadPackage(eq(uploader));
    doThrow(new PackingException("")).when(submitterMain).callLauncherRunner(Mockito.any(Config.class));
    submitterMain.submitTopology();
}