Examples with PackingPlan com.twitter.heron.spi.packing.PackingPlan used on opensource projects

Example 71 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class HeronMasterDriverTest method scheduleHeronWorkersFailsOnDuplicateRequest.

@Test(expected = HeronMasterDriver.ContainerAllocationException.class)
public void scheduleHeronWorkersFailsOnDuplicateRequest() throws Exception {
    PackingPlan packingPlan = PackingTestUtils.testPackingPlan("test", new RoundRobinPacking());
    spyDriver.scheduleHeronWorkers(packingPlan);
    verify(spyDriver, times(1)).requestContainerForWorker(eq(1), anyHeronWorker());
    verify(mockRequestor, times(1)).submit(any(EvaluatorRequest.class));
    PackingPlan.ContainerPlan duplicatePlan = PackingTestUtils.testContainerPlan(1);
    Set<PackingPlan.ContainerPlan> toBeAddedContainerPlans = new HashSet<>();
    toBeAddedContainerPlans.add(duplicatePlan);
    spyDriver.scheduleHeronWorkers(toBeAddedContainerPlans);
}

Example 72 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class MesosLauncherTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    MesosLauncher launcher = Mockito.spy(MesosLauncher.class);
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    Config config = Mockito.mock(Config.class);
    Mockito.doReturn("working-dir").when(config).getStringValue(MesosContext.SCHEDULER_WORKING_DIRECTORY);
    Config runtime = Mockito.mock(Config.class);
    launcher.initialize(config, runtime);
    // Failed to setup working dir
    Mockito.doReturn(false).when(launcher).setupWorkingDirectory();
    Assert.assertFalse(launcher.launch(packingPlan));
    Mockito.doReturn(true).when(launcher).setupWorkingDirectory();
    String[] mockCommand = new String[] { "mock", "scheduler", "command" };
    Mockito.doReturn(mockCommand).when(launcher).getSchedulerCommand();
    // Failed to spwan the process
    Mockito.doReturn(null).when(launcher).startScheduler(mockCommand);
    Assert.assertFalse(launcher.launch(packingPlan));
    // Happy path
    Process p = Mockito.mock(Process.class);
    Mockito.doReturn(p).when(launcher).startScheduler(mockCommand);
    Assert.assertTrue(launcher.launch(packingPlan));
}

Example 73 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class MesosSchedulerTest method testGetBaseContainer.

@Test
public void testGetBaseContainer() throws Exception {
    final double CPU = 0.5;
    final ByteAmount MEM = ByteAmount.fromMegabytes(100);
    final ByteAmount DISK = ByteAmount.fromMegabytes(100);
    Resource containerResources = new Resource(CPU, MEM, DISK);
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(0, new HashSet<PackingPlan.InstancePlan>(), containerResources);
    Set<PackingPlan.ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlan);
    PackingPlan packingPlan = new PackingPlan(TOPOLOGY_NAME, containerPlans);
    BaseContainer container = scheduler.getBaseContainer(0, packingPlan);
    // Assert we have constructed the correct BaseContainer structure
    Assert.assertEquals(ROLE, container.runAsUser);
    Assert.assertEquals(CPU, container.cpu, 0.01);
    Assert.assertEquals(MEM, ByteAmount.fromMegabytes(((Double) container.memInMB).longValue()));
    Assert.assertEquals(DISK, ByteAmount.fromMegabytes(((Double) container.diskInMB).longValue()));
    Assert.assertEquals(SchedulerUtils.PORTS_REQUIRED_FOR_EXECUTOR, container.ports);
    Assert.assertEquals(2, container.dependencies.size());
    Assert.assertTrue(container.dependencies.contains(CORE_PACKAGE_URI));
    Assert.assertTrue(container.dependencies.contains(TOPOLOGY_PACKAGE_URI));
    Assert.assertTrue(container.environmentVariables.isEmpty());
    Assert.assertTrue(container.name.startsWith("container_0"));
    // Convert to JSON
    String str = container.toString();
    String json = BaseContainer.getJobDefinitionInJSON(container);
    Assert.assertEquals(json, str);
    // Convert the JSON back to BaseContainer
    BaseContainer newContainer = BaseContainer.getJobFromJSONString(json);
    Assert.assertEquals(json, BaseContainer.getJobDefinitionInJSON(newContainer));
}

Example 74 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SlurmLauncherTest method testLaunch.

/**
   * Test slurm scheduler launcher
   */
@Test
public void testLaunch() throws Exception {
    Config config = createRunnerConfig();
    Config runtime = Mockito.mock(Config.class);
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    PackingPlan plan = new PackingPlan("plan.id", new HashSet<PackingPlan.ContainerPlan>());
    PowerMockito.spy(SlurmContext.class);
    PowerMockito.doReturn(WORKING_DIRECTORY).when(SlurmContext.class, "workingDirectory", config);
    LauncherUtils mockLauncherUtils = Mockito.mock(LauncherUtils.class);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    SlurmLauncher slurmLauncher = Mockito.spy(new SlurmLauncher());
    slurmLauncher.initialize(config, runtime);
    SlurmScheduler slurmScheduler = Mockito.spy(new SlurmScheduler());
    PowerMockito.doReturn(true).when(slurmScheduler).onSchedule(plan);
    // Failed to download
    Mockito.doReturn(false).when(slurmLauncher).setupWorkingDirectory();
    Assert.assertFalse(slurmLauncher.launch(packingPlan));
    Mockito.verify(slurmLauncher).setupWorkingDirectory();
    // Failed to schedule
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(false);
    PowerMockito.doReturn(true).when(slurmLauncher).setupWorkingDirectory();
    Assert.assertFalse(slurmLauncher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    // happy path
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(true);
    Assert.assertTrue(slurmLauncher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(slurmLauncher, Mockito.times(3)).launch(Mockito.any(PackingPlan.class));
    Mockito.verify(mockLauncherUtils, Mockito.times(2)).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    slurmLauncher.close();
}

Example 75 with PackingPlan

use of com.twitter.heron.spi.packing.PackingPlan in project heron by twitter.

the class SchedulerMain method runScheduler.

/**
   * Run the scheduler.
   * It is a blocking call, and it will return in 2 cases:
   * 1. The topology is requested to kill
   * 2. Unexpected exceptions happen
   *
   * @return true if scheduled successfully
   */
public boolean runScheduler() {
    IScheduler scheduler = null;
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    try {
        // create an instance of state manager
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        LOG.log(Level.SEVERE, "Failed to instantiate instances using config: " + config, e);
        return false;
    }
    SchedulerServer server = null;
    boolean isSuccessful = false;
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // get a packed plan and schedule it
        PackingPlans.PackingPlan serializedPackingPlan = adaptor.getPackingPlan(topology.getName());
        if (serializedPackingPlan == null) {
            LOG.log(Level.SEVERE, "Failed to fetch PackingPlan for topology:{0} from the state manager", topology.getName());
            return false;
        }
        LOG.log(Level.INFO, "Packing plan fetched from state: {0}", serializedPackingPlan);
        PackingPlan packedPlan = new PackingPlanProtoDeserializer().fromProto(serializedPackingPlan);
        // build the runtime config
        LauncherUtils launcherUtils = LauncherUtils.getInstance();
        Config runtime = Config.newBuilder().putAll(launcherUtils.createPrimaryRuntime(topology)).putAll(launcherUtils.createAdaptorRuntime(adaptor)).put(Key.SCHEDULER_SHUTDOWN, getShutdown()).put(Key.SCHEDULER_PROPERTIES, properties).build();
        Config ytruntime = launcherUtils.createConfigWithPackingDetails(runtime, packedPlan);
        // invoke scheduler
        scheduler = launcherUtils.getSchedulerInstance(config, ytruntime);
        if (scheduler == null) {
            return false;
        }
        isSuccessful = scheduler.onSchedule(packedPlan);
        if (!isSuccessful) {
            LOG.severe("Failed to schedule topology");
            return false;
        }
        // Failures in server initialization throw exceptions
        // get the scheduler server endpoint for receiving requests
        server = getServer(ytruntime, scheduler, schedulerServerPort);
        // start the server to manage runtime requests
        server.start();
        // write the scheduler location to state manager
        // Make sure it happens after IScheduler.onScheduler
        isSuccessful = SchedulerUtils.setSchedulerLocation(runtime, String.format("%s:%d", server.getHost(), server.getPort()), scheduler);
        if (isSuccessful) {
            // wait until kill request or some interrupt occurs if the scheduler starts successfully
            LOG.info("Waiting for termination... ");
            Runtime.schedulerShutdown(ytruntime).await();
        }
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Failed to start server", e);
        return false;
    } finally {
        // Clean the resources
        if (server != null) {
            server.stop();
        }
        // 4. Close the resources
        SysUtils.closeIgnoringExceptions(scheduler);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
    return isSuccessful;
}