Examples with IStateManager com.twitter.heron.spi.statemgr.IStateManager used on opensource projects

Example 1 with IStateManager

use of com.twitter.heron.spi.statemgr.IStateManager in project heron by twitter.

the class SchedulerMain method runScheduler.

/**
   * Run the scheduler.
   * It is a blocking call, and it will return in 2 cases:
   * 1. The topology is requested to kill
   * 2. Unexpected exceptions happen
   *
   * @return true if scheduled successfully
   */
public boolean runScheduler() {
    IScheduler scheduler = null;
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    try {
        // create an instance of state manager
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        LOG.log(Level.SEVERE, "Failed to instantiate instances using config: " + config, e);
        return false;
    }
    SchedulerServer server = null;
    boolean isSuccessful = false;
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // get a packed plan and schedule it
        PackingPlans.PackingPlan serializedPackingPlan = adaptor.getPackingPlan(topology.getName());
        if (serializedPackingPlan == null) {
            LOG.log(Level.SEVERE, "Failed to fetch PackingPlan for topology:{0} from the state manager", topology.getName());
            return false;
        }
        LOG.log(Level.INFO, "Packing plan fetched from state: {0}", serializedPackingPlan);
        PackingPlan packedPlan = new PackingPlanProtoDeserializer().fromProto(serializedPackingPlan);
        // build the runtime config
        LauncherUtils launcherUtils = LauncherUtils.getInstance();
        Config runtime = Config.newBuilder().putAll(launcherUtils.createPrimaryRuntime(topology)).putAll(launcherUtils.createAdaptorRuntime(adaptor)).put(Key.SCHEDULER_SHUTDOWN, getShutdown()).put(Key.SCHEDULER_PROPERTIES, properties).build();
        Config ytruntime = launcherUtils.createConfigWithPackingDetails(runtime, packedPlan);
        // invoke scheduler
        scheduler = launcherUtils.getSchedulerInstance(config, ytruntime);
        if (scheduler == null) {
            return false;
        }
        isSuccessful = scheduler.onSchedule(packedPlan);
        if (!isSuccessful) {
            LOG.severe("Failed to schedule topology");
            return false;
        }
        // Failures in server initialization throw exceptions
        // get the scheduler server endpoint for receiving requests
        server = getServer(ytruntime, scheduler, schedulerServerPort);
        // start the server to manage runtime requests
        server.start();
        // write the scheduler location to state manager
        // Make sure it happens after IScheduler.onScheduler
        isSuccessful = SchedulerUtils.setSchedulerLocation(runtime, String.format("%s:%d", server.getHost(), server.getPort()), scheduler);
        if (isSuccessful) {
            // wait until kill request or some interrupt occurs if the scheduler starts successfully
            LOG.info("Waiting for termination... ");
            Runtime.schedulerShutdown(ytruntime).await();
        }
    } catch (IOException e) {
        LOG.log(Level.SEVERE, "Failed to start server", e);
        return false;
    } finally {
        // Clean the resources
        if (server != null) {
            server.stop();
        }
        // 4. Close the resources
        SysUtils.closeIgnoringExceptions(scheduler);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
    return isSuccessful;
}

Example 2 with IStateManager

use of com.twitter.heron.spi.statemgr.IStateManager in project heron by twitter.

the class SubmitterMain method submitTopology.

/**
   * Submit a topology
   * 1. Instantiate necessary resources
   * 2. Valid whether it is legal to submit a topology
   * 3. Call LauncherRunner
   *
   */
public void submitTopology() throws TopologySubmissionException {
    // build primary runtime config first
    Config primaryRuntime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(topology)).build();
    // call launcher directly here if in dry-run mode
    if (Context.dryRun(config)) {
        callLauncherRunner(primaryRuntime);
        return;
    }
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    // Create an instance of the launcher class
    String launcherClass = Context.launcherClass(config);
    ILauncher launcher;
    // create an instance of the uploader class
    String uploaderClass = Context.uploaderClass(config);
    IUploader uploader;
    // create an instance of state manager
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologySubmissionException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // create an instance of launcher
    try {
        launcher = ReflectionUtils.newInstance(launcherClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new LauncherException(String.format("Failed to instantiate launcher class '%s'", launcherClass), e);
    }
    // create an instance of uploader
    try {
        uploader = ReflectionUtils.newInstance(uploaderClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new UploaderException(String.format("Failed to instantiate uploader class '%s'", uploaderClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the state manager
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        // Check if topology is already running
        validateSubmit(adaptor, topology.getName());
        LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName());
        Config runtimeWithoutPackageURI = Config.newBuilder().putAll(primaryRuntime).putAll(LauncherUtils.getInstance().createAdaptorRuntime(adaptor)).put(Key.LAUNCHER_CLASS_INSTANCE, launcher).build();
        PackingPlan packingPlan = LauncherUtils.getInstance().createPackingPlan(config, runtimeWithoutPackageURI);
        // The packing plan might call for a number of containers different than the config
        // settings. If that's the case we need to modify the configs to match.
        runtimeWithoutPackageURI = updateNumContainersIfNeeded(runtimeWithoutPackageURI, topology, packingPlan);
        // If the packing plan is valid we will upload necessary packages
        URI packageURI = uploadPackage(uploader);
        // Update the runtime config with the packageURI
        Config runtimeAll = Config.newBuilder().putAll(runtimeWithoutPackageURI).put(Key.TOPOLOGY_PACKAGE_URI, packageURI).build();
        callLauncherRunner(runtimeAll);
    } catch (LauncherException | PackingException e) {
        // we undo uploading of topology package only if launcher fails to
        // launch topology, which will throw LauncherException or PackingException
        uploader.undo();
        throw e;
    } finally {
        SysUtils.closeIgnoringExceptions(uploader);
        SysUtils.closeIgnoringExceptions(launcher);
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 3 with IStateManager

use of com.twitter.heron.spi.statemgr.IStateManager in project heron by twitter.

the class RuntimeManagerMain method manageTopology.

/**
   * Manager a topology
   * 1. Instantiate necessary resources
   * 2. Valid whether the runtime management is legal
   * 3. Complete the runtime management for a specific command
   *
   */
public void manageTopology() throws TopologyRuntimeManagementException, TMasterException, PackingException {
    String topologyName = Context.topologyName(config);
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    IStateManager statemgr;
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new TopologyRuntimeManagementException(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the statemgr
        statemgr.initialize(config);
        // TODO(mfu): timeout should read from config
        SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000);
        validateRuntimeManage(adaptor, topologyName);
        // 2. Try to manage topology if valid
        // invoke the appropriate command to manage the topology
        LOG.log(Level.FINE, "Topology: {0} to be {1}ed", new Object[] { topologyName, command });
        // build the runtime config
        Config runtime = Config.newBuilder().put(Key.TOPOLOGY_NAME, Context.topologyName(config)).put(Key.SCHEDULER_STATE_MANAGER_ADAPTOR, adaptor).build();
        // Create a ISchedulerClient basing on the config
        ISchedulerClient schedulerClient = getSchedulerClient(runtime);
        callRuntimeManagerRunner(runtime, schedulerClient);
    } finally {
        // 3. Do post work basing on the result
        // Currently nothing to do here
        // 4. Close the resources
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}

Example 4 with IStateManager

use of com.twitter.heron.spi.statemgr.IStateManager in project heron by twitter.

the class MetricsCacheManager method start.

public void start() throws Exception {
    // 1. Do prepare work
    // create an instance of state manager
    String statemgrClass = Context.stateManagerClass(config);
    LOG.info("Context.stateManagerClass " + statemgrClass);
    IStateManager statemgr;
    try {
        statemgr = ReflectionUtils.newInstance(statemgrClass);
    } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
        throw new Exception(String.format("Failed to instantiate state manager class '%s'", statemgrClass), e);
    }
    // Put it in a try block so that we can always clean resources
    try {
        // initialize the statemgr
        statemgr.initialize(config);
        statemgr.setMetricsCacheLocation(metricsCacheLocation, topologyName);
        LOG.info("metricsCacheLocation " + metricsCacheLocation.toString());
        LOG.info("topologyName " + topologyName.toString());
        LOG.info("Starting Metrics Cache HTTP Server");
        metricsCacheManagerHttpServer.start();
        // 2. The MetricsCacheServer would run in the main thread
        // We do it in the final step since it would await the main thread
        LOG.info("Starting Metrics Cache Server");
        metricsCacheManagerServer.start();
        metricsCacheManagerServerLoop.loop();
    } finally {
        // 3. Do post work basing on the result
        // Currently nothing to do here
        // 4. Close the resources
        SysUtils.closeIgnoringExceptions(statemgr);
    }
}