Examples with PackingPlan org.apache.heron.spi.packing.PackingPlan used on opensource projects

Example 36 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class MesosLauncherTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    MesosLauncher launcher = Mockito.spy(MesosLauncher.class);
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    Config config = Mockito.mock(Config.class);
    Mockito.doReturn("working-dir").when(config).getStringValue(MesosContext.SCHEDULER_WORKING_DIRECTORY);
    Config runtime = Mockito.mock(Config.class);
    launcher.initialize(config, runtime);
    // Failed to setup working dir
    Mockito.doReturn(false).when(launcher).setupWorkingDirectory();
    Assert.assertFalse(launcher.launch(packingPlan));
    Mockito.doReturn(true).when(launcher).setupWorkingDirectory();
    String[] mockCommand = new String[] { "mock", "scheduler", "command" };
    Mockito.doReturn(mockCommand).when(launcher).getSchedulerCommand();
    // Failed to spwan the process
    Mockito.doReturn(null).when(launcher).startScheduler(mockCommand);
    Assert.assertFalse(launcher.launch(packingPlan));
    // Happy path
    Process p = Mockito.mock(Process.class);
    Mockito.doReturn(p).when(launcher).startScheduler(mockCommand);
    Assert.assertTrue(launcher.launch(packingPlan));
}

Example 37 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class HeronMasterDriverTest method killWorkersTerminatesSpecificContainers.

@Test
public void killWorkersTerminatesSpecificContainers() throws Exception {
    int numContainers = 5;
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    for (int id = 0; id < numContainers; id++) {
        containers.add(PackingTestUtils.testContainerPlan(id));
    }
    PackingPlan packingPlan = new PackingPlan("packing", containers);
    spyDriver.scheduleHeronWorkers(packingPlan);
    for (int id = 0; id < numContainers; id++) {
        verify(spyDriver, times(1)).requestContainerForWorker(eq(id), anyHeronWorker());
        assertTrue(spyDriver.lookupByContainerPlan(id).isPresent());
    }
    verify(mockRequestor, times(numContainers)).submit(any(EvaluatorRequest.class));
    AllocatedEvaluator[] mockEvaluators = createApplicationWithContainers(numContainers);
    Set<PackingPlan.ContainerPlan> containersTobeDeleted = new HashSet<>();
    containersTobeDeleted.add(PackingTestUtils.testContainerPlan(2));
    containersTobeDeleted.add(PackingTestUtils.testContainerPlan(3));
    spyDriver.killWorkers(containersTobeDeleted);
    for (int id = 0; id < numContainers; id++) {
        if (id == 2 || id == 3) {
            verify(mockEvaluators[id], times(1)).close();
            assertFalse(spyDriver.lookupByContainerPlan(id).isPresent());
            assertFalse(spyDriver.lookupByEvaluatorId("e" + id).isPresent());
            continue;
        }
        verify(mockEvaluators[id], never()).close();
        assertTrue(spyDriver.lookupByContainerPlan(id).isPresent());
        assertTrue(spyDriver.lookupByEvaluatorId("e" + id).isPresent());
    }
}

Example 38 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class HeronMasterDriverTest method scheduleHeronWorkersFailsOnDuplicateRequest.

@Test(expected = HeronMasterDriver.ContainerAllocationException.class)
public void scheduleHeronWorkersFailsOnDuplicateRequest() throws Exception {
    PackingPlan packingPlan = PackingTestUtils.testPackingPlan("test", new RoundRobinPacking());
    spyDriver.scheduleHeronWorkers(packingPlan);
    verify(spyDriver, times(1)).requestContainerForWorker(eq(1), anyHeronWorker());
    verify(mockRequestor, times(1)).submit(any(EvaluatorRequest.class));
    PackingPlan.ContainerPlan duplicatePlan = PackingTestUtils.testContainerPlan(1);
    Set<PackingPlan.ContainerPlan> toBeAddedContainerPlans = new HashSet<>();
    toBeAddedContainerPlans.add(duplicatePlan);
    spyDriver.scheduleHeronWorkers(toBeAddedContainerPlans);
}

Example 39 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class PackingPlanProviderTest method refreshesPackingPlanOnUpdate.

@Test
public void refreshesPackingPlanOnUpdate() {
    PackingPlans.PackingPlan proto = PackingTestUtils.testProtoPackingPlan(topologyName, new RoundRobinPacking());
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    when(adaptor.getPackingPlan(topologyName)).thenReturn(proto);
    PackingPlanProvider provider = new PackingPlanProvider(adaptor, eventManager, topologyName);
    PackingPlan packing = provider.get();
    Assert.assertEquals(1, packing.getContainers().size());
    provider.onEvent(new TopologyUpdate(null, null));
    provider.get();
    verify(adaptor, times(2)).getPackingPlan(topologyName);
}

Example 40 with PackingPlan

use of org.apache.heron.spi.packing.PackingPlan in project heron by twitter.

the class ScaleUpResolver method resolve.

@Override
public Collection<Action> resolve(Collection<Diagnosis> diagnosis) {
    List<Action> actions = new ArrayList<>();
    DiagnosisTable table = DiagnosisTable.of(diagnosis);
    table = table.type(DIAGNOSIS_UNDER_PROVISIONING.text());
    if (table.size() == 0) {
        LOG.fine("No under-previsioning diagnosis present, ending as there's nothing to fix");
        return actions;
    }
    // Scale the first assigned component
    Diagnosis diagnoses = table.first();
    // verify diagnoses instance is valid
    if (diagnoses.assignments().isEmpty()) {
        LOG.warning(String.format("Diagnosis %s is missing assignments", diagnoses.id()));
        return actions;
    }
    String component = diagnoses.assignments().iterator().next();
    int newParallelism = computeScaleUpFactor(component);
    Map<String, Integer> changeRequest = new HashMap<>();
    changeRequest.put(component, newParallelism);
    PackingPlan currentPackingPlan = packingPlanProvider.get();
    PackingPlan newPlan = buildNewPackingPlan(changeRequest, currentPackingPlan);
    if (newPlan == null) {
        return null;
    }
    Scheduler.UpdateTopologyRequest updateTopologyRequest = Scheduler.UpdateTopologyRequest.newBuilder().setCurrentPackingPlan(getSerializedPlan(currentPackingPlan)).setProposedPackingPlan(getSerializedPlan(newPlan)).build();
    LOG.info("Sending Updating topology request: " + updateTopologyRequest);
    if (!schedulerClient.updateTopology(updateTopologyRequest)) {
        throw new RuntimeException(String.format("Failed to update topology with Scheduler, " + "updateTopologyRequest=%s", updateTopologyRequest));
    }
    LOG.info("Scheduler updated topology successfully.");
    LOG.info("Broadcasting topology update event");
    TopologyUpdate action = new TopologyUpdate(context.checkpoint(), Collections.singletonList(component));
    eventManager.onEvent(action);
    actions.add(action);
    return actions;
}