Examples with SchedulerStateManagerAdaptor org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor used on opensource projects

Example 21 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class SchedulerUtilsTest method persistUpdatedPackingPlanWillUpdatesStateManager.

@Test
public void persistUpdatedPackingPlanWillUpdatesStateManager() {
    SchedulerStateManagerAdaptor adaptor = Mockito.mock(SchedulerStateManagerAdaptor.class);
    Mockito.when(adaptor.updatePackingPlan(Mockito.any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME))).thenReturn(true);
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(PackingTestUtils.testContainerPlan(1, 0, 1, 2));
    PackingPlan packing = new PackingPlan("id", containers);
    SchedulerUtils.persistUpdatedPackingPlan(TOPOLOGY_NAME, packing, adaptor);
    Mockito.verify(adaptor).updatePackingPlan(Mockito.any(PackingPlans.PackingPlan.class), eq(TOPOLOGY_NAME));
}

Example 22 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class SubmitterMain method submitTopology.

/**
 * Submit a topology
 * 1. Instantiate necessary resources
 * 2. Valid whether it is legal to submit a topology
 * 3. Call LauncherRunner
 */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // initialize the uploader
        uploader.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 23 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class RuntimeManagerRunner method restartTopologyHandler.

/**
 * Handler to restart a topology
 */
@VisibleForTesting
void restartTopologyHandler(String topologyName) throws TopologyRuntimeManagementException {
    assert !potentialStaleExecutionData;
    Integer containerId = Context.topologyContainerId(config);
    Scheduler.RestartTopologyRequest restartTopologyRequest = Scheduler.RestartTopologyRequest.newBuilder().setTopologyName(topologyName).setContainerIndex(containerId).build();
    // i.e. TManagerLocation does not exist
    if (containerId == -1 || containerId == 0) {
        // get the instance of state manager to clean state
        SchedulerStateManagerAdaptor stateManager = Runtime.schedulerStateManagerAdaptor(runtime);
        Boolean result = stateManager.deleteTManagerLocation(topologyName);
        if (result == null || !result) {
            throw new TopologyRuntimeManagementException("Failed to clear TManager location. Check whether TManager set it correctly.");
        }
    }
    if (!schedulerClient.restartTopology(restartTopologyRequest)) {
        throw new TopologyRuntimeManagementException(String.format("Failed to restart topology '%s'", topologyName));
    }
    // Clean the connection when we are done.
    LOG.fine("Scheduler restarted topology successfully.");
}

Example 24 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class RuntimeManagerRunner method updateTopologyComponentParallelism.

@VisibleForTesting
void updateTopologyComponentParallelism(String topologyName, String newParallelism) throws TopologyRuntimeManagementException, PackingException, UpdateDryRunResponse {
    LOG.fine(String.format("updateTopologyHandler called for %s with %s", topologyName, newParallelism));
    Map<String, Integer> changeRequests = parseNewParallelismParam(newParallelism);
    SchedulerStateManagerAdaptor manager = Runtime.schedulerStateManagerAdaptor(runtime);
    TopologyAPI.Topology topology = manager.getTopology(topologyName);
    PackingPlans.PackingPlan currentPlan = manager.getPackingPlan(topologyName);
    if (!parallelismChangeDetected(currentPlan, changeRequests)) {
        throw new TopologyRuntimeManagementException(String.format("The component parallelism request (%s) is the same as the " + "current topology parallelism. Not taking action.", newParallelism));
    }
    PackingPlans.PackingPlan proposedPlan = buildNewPackingPlan(currentPlan, changeRequests, null, topology);
    sendUpdateRequest(topology, changeRequests, currentPlan, proposedPlan);
}

Example 25 with SchedulerStateManagerAdaptor

use of org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor in project heron by twitter.

the class SchedulerClientFactoryTest method testGetServiceSchedulerClientFail.

@Test(expected = SchedulerException.class)
public void testGetServiceSchedulerClientFail() throws Exception {
    // Instantiate mock objects
    Config config = Mockito.mock(Config.class);
    Config runtime = Mockito.mock(Config.class);
    SchedulerStateManagerAdaptor statemgr = Mockito.mock(SchedulerStateManagerAdaptor.class);
    // Get a ServiceSchedulerClient
    Mockito.when(config.getBooleanValue(Key.SCHEDULER_IS_SERVICE)).thenReturn(true);
    // Mock the runtime object
    Mockito.when(runtime.get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR)).thenReturn(statemgr);
    Mockito.when(runtime.getStringValue(Key.TOPOLOGY_NAME)).thenReturn(TOPOLOGY_NAME);
    // Failed to getSchedulerLocation
    Mockito.when(statemgr.getSchedulerLocation(Mockito.eq(TOPOLOGY_NAME))).thenReturn(null);
    try {
        new SchedulerClientFactory(config, runtime).getSchedulerClient();
    } finally {
        Mockito.verify(statemgr).getSchedulerLocation(Mockito.eq(TOPOLOGY_NAME));
    }
}