Examples with Scheduler org.apache.heron.proto.scheduler.Scheduler used on opensource projects

Example 6 with Scheduler

use of org.apache.heron.proto.scheduler.Scheduler in project heron by twitter.

the class RuntimeManagerRunner method sendUpdateRequest.

void sendUpdateRequest(TopologyAPI.Topology topology, Map<String, Integer> changeRequests, PackingPlans.PackingPlan currentPlan, PackingPlans.PackingPlan proposedPlan) {
    if (Context.dryRun(config)) {
        PackingPlanProtoDeserializer deserializer = new PackingPlanProtoDeserializer();
        PackingPlan oldPlan = deserializer.fromProto(currentPlan);
        PackingPlan newPlan = deserializer.fromProto(proposedPlan);
        throw new UpdateDryRunResponse(topology, config, newPlan, oldPlan, changeRequests);
    }
    Scheduler.UpdateTopologyRequest updateTopologyRequest = Scheduler.UpdateTopologyRequest.newBuilder().setCurrentPackingPlan(currentPlan).setProposedPackingPlan(proposedPlan).build();
    LOG.fine("Sending Updating topology request: " + updateTopologyRequest);
    if (!schedulerClient.updateTopology(updateTopologyRequest)) {
        throw new TopologyRuntimeManagementException("Failed to update topology with Scheduler, updateTopologyRequest=" + updateTopologyRequest + "The topology can be in a strange stage. " + "Please check carefully or redeploy the topology !!");
    }
}

Example 7 with Scheduler

use of org.apache.heron.proto.scheduler.Scheduler in project heron by twitter.

the class SchedulerUtils method setSchedulerLocation.

/**
 * Set the location of scheduler for other processes to discover
 *
 * @param runtime the runtime configuration
 * @param schedulerEndpoint the endpoint that scheduler listens for receives requests
 * @param scheduler the IScheduler to provide more info
 */
public static boolean setSchedulerLocation(Config runtime, String schedulerEndpoint, IScheduler scheduler) {
    // Set scheduler location to host:port by default. Overwrite scheduler location if behind DNS.
    Scheduler.SchedulerLocation.Builder builder = Scheduler.SchedulerLocation.newBuilder().setTopologyName(Runtime.topologyName(runtime)).setHttpEndpoint(schedulerEndpoint);
    // Set the job link in SchedulerLocation if any
    List<String> jobLinks = scheduler.getJobLinks();
    // Check whether IScheduler provides valid job link
    if (jobLinks != null) {
        builder.addAllJobPageLink(jobLinks);
    }
    Scheduler.SchedulerLocation location = builder.build();
    LOG.log(Level.INFO, "Setting Scheduler locations: {0}", location);
    SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
    Boolean result = statemgr.setSchedulerLocation(location, Runtime.topologyName(runtime));
    if (result == null || !result) {
        LOG.severe("Failed to set Scheduler location");
        return false;
    }
    return true;
}

Example 8 with Scheduler

use of org.apache.heron.proto.scheduler.Scheduler in project heron by twitter.

the class AuroraSchedulerTest method testOnRestart.

@Test
public void testOnRestart() throws Exception {
    Config mockConfig = Mockito.mock(Config.class);
    PowerMockito.mockStatic(Config.class);
    when(Config.toClusterMode(mockConfig)).thenReturn(mockConfig);
    AuroraController controller = Mockito.mock(AuroraController.class);
    doReturn(controller).when(scheduler).getController();
    scheduler.initialize(mockConfig, Mockito.mock(Config.class));
    // Construct the RestartTopologyRequest
    int containerToRestart = 1;
    Scheduler.RestartTopologyRequest restartTopologyRequest = Scheduler.RestartTopologyRequest.newBuilder().setTopologyName(TOPOLOGY_NAME).setContainerIndex(containerToRestart).build();
    // Failed to kill job via controller
    doReturn(false).when(controller).restart(containerToRestart);
    Assert.assertFalse(scheduler.onRestart(restartTopologyRequest));
    Mockito.verify(controller).restart(containerToRestart);
    // Happy path
    doReturn(true).when(controller).restart(containerToRestart);
    assertTrue(scheduler.onRestart(restartTopologyRequest));
    Mockito.verify(controller, Mockito.times(2)).restart(containerToRestart);
}

Example 9 with Scheduler

use of org.apache.heron.proto.scheduler.Scheduler in project heron by twitter.

the class ScaleUpResolver method resolve.

@Override
public Collection<Action> resolve(Collection<Diagnosis> diagnosis) {
    List<Action> actions = new ArrayList<>();
    DiagnosisTable table = DiagnosisTable.of(diagnosis);
    table = table.type(DIAGNOSIS_UNDER_PROVISIONING.text());
    if (table.size() == 0) {
        LOG.fine("No under-previsioning diagnosis present, ending as there's nothing to fix");
        return actions;
    }
    // Scale the first assigned component
    Diagnosis diagnoses = table.first();
    // verify diagnoses instance is valid
    if (diagnoses.assignments().isEmpty()) {
        LOG.warning(String.format("Diagnosis %s is missing assignments", diagnoses.id()));
        return actions;
    }
    String component = diagnoses.assignments().iterator().next();
    int newParallelism = computeScaleUpFactor(component);
    Map<String, Integer> changeRequest = new HashMap<>();
    changeRequest.put(component, newParallelism);
    PackingPlan currentPackingPlan = packingPlanProvider.get();
    PackingPlan newPlan = buildNewPackingPlan(changeRequest, currentPackingPlan);
    if (newPlan == null) {
        return null;
    }
    Scheduler.UpdateTopologyRequest updateTopologyRequest = Scheduler.UpdateTopologyRequest.newBuilder().setCurrentPackingPlan(getSerializedPlan(currentPackingPlan)).setProposedPackingPlan(getSerializedPlan(newPlan)).build();
    LOG.info("Sending Updating topology request: " + updateTopologyRequest);
    if (!schedulerClient.updateTopology(updateTopologyRequest)) {
        throw new RuntimeException(String.format("Failed to update topology with Scheduler, " + "updateTopologyRequest=%s", updateTopologyRequest));
    }
    LOG.info("Scheduler updated topology successfully.");
    LOG.info("Broadcasting topology update event");
    TopologyUpdate action = new TopologyUpdate(context.checkpoint(), Collections.singletonList(component));
    eventManager.onEvent(action);
    actions.add(action);
    return actions;
}