Search in sources :

Example 1 with Resources

use of co.cask.cdap.api.Resources in project cdap by caskdata.

the class ServiceSpecificationCodec method decodeOldSpec.

private ServiceSpecification decodeOldSpec(JsonObject json) {
    String className = json.get("classname").getAsString();
    TwillSpecification twillSpec = twillSpecificationAdapter.fromJson(json.get("spec").getAsString()).getTwillSpecification();
    Map<String, HttpServiceHandlerSpecification> handlers = Maps.newHashMap();
    RuntimeSpecification handlerSpec = twillSpec.getRunnables().get(twillSpec.getName());
    Map<String, String> configs = handlerSpec.getRunnableSpecification().getConfigs();
    // Get the class names of all handlers. It is stored in the handler runnable spec configs
    List<String> handlerClasses = GSON.fromJson(configs.get("service.runnable.handlers"), new TypeToken<List<String>>() {
    }.getType());
    List<JsonObject> handlerSpecs = GSON.fromJson(configs.get("service.runnable.handler.spec"), new TypeToken<List<JsonObject>>() {
    }.getType());
    for (int i = 0; i < handlerClasses.size(); i++) {
        String handlerClass = handlerClasses.get(i);
        JsonObject spec = handlerSpecs.get(i);
        Map<String, String> properties = GSON.fromJson(spec.get("properties"), new TypeToken<Map<String, String>>() {
        }.getType());
        // Reconstruct the HttpServiceSpecification. However there is no way to determine the datasets or endpoints
        // as it is not recorded in old spec. It's ok since the spec is only used to load data from MDS during redeploy.
        handlers.put(spec.get("name").getAsString(), new HttpServiceHandlerSpecification(handlerClass, spec.get("name").getAsString(), spec.get("description").getAsString(), properties, ImmutableSet.<String>of(), ImmutableList.<ServiceHttpEndpoint>of()));
    }
    ResourceSpecification resourceSpec = handlerSpec.getResourceSpecification();
    return new ServiceSpecification(className, twillSpec.getName(), twillSpec.getName(), handlers, new Resources(resourceSpec.getMemorySize(), resourceSpec.getVirtualCores()), resourceSpec.getInstances());
}
Also used : ServiceHttpEndpoint(co.cask.cdap.api.service.http.ServiceHttpEndpoint) ServiceSpecification(co.cask.cdap.api.service.ServiceSpecification) JsonObject(com.google.gson.JsonObject) ResourceSpecification(org.apache.twill.api.ResourceSpecification) HttpServiceHandlerSpecification(co.cask.cdap.api.service.http.HttpServiceHandlerSpecification) RuntimeSpecification(org.apache.twill.api.RuntimeSpecification) TwillSpecification(org.apache.twill.api.TwillSpecification) ServiceHttpEndpoint(co.cask.cdap.api.service.http.ServiceHttpEndpoint) TypeToken(com.google.common.reflect.TypeToken) Resources(co.cask.cdap.api.Resources)

Example 2 with Resources

use of co.cask.cdap.api.Resources in project cdap by caskdata.

the class ServiceSpecificationCodec method deserialize.

@Override
public ServiceSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = (JsonObject) json;
    if (isOldSpec(jsonObj)) {
        return decodeOldSpec(jsonObj);
    }
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, HttpServiceHandlerSpecification> handlers = deserializeMap(jsonObj.get("handlers"), context, HttpServiceHandlerSpecification.class);
    Resources resources = context.deserialize(jsonObj.get("resources"), Resources.class);
    int instances = jsonObj.get("instances").getAsInt();
    return new ServiceSpecification(className, name, description, handlers, resources, instances);
}
Also used : ServiceSpecification(co.cask.cdap.api.service.ServiceSpecification) JsonObject(com.google.gson.JsonObject) Resources(co.cask.cdap.api.Resources) HttpServiceHandlerSpecification(co.cask.cdap.api.service.http.HttpServiceHandlerSpecification) ServiceHttpEndpoint(co.cask.cdap.api.service.http.ServiceHttpEndpoint)

Example 3 with Resources

use of co.cask.cdap.api.Resources in project cdap by caskdata.

the class DistributedWorkflowProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(LaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
    WorkflowSpecification spec = program.getApplicationSpecification().getWorkflows().get(program.getName());
    List<ClassAcceptor> acceptors = new ArrayList<>();
    // Only interested in MapReduce and Spark nodes
    Set<SchedulableProgramType> runnerTypes = EnumSet.of(SchedulableProgramType.MAPREDUCE, SchedulableProgramType.SPARK);
    for (WorkflowActionNode node : Iterables.filter(spec.getNodeIdMap().values(), WorkflowActionNode.class)) {
        // For each type, we only need one node to setup the launch context
        ScheduleProgramInfo programInfo = node.getProgram();
        if (!runnerTypes.remove(programInfo.getProgramType())) {
            continue;
        }
        // Find the ProgramRunner of the given type and setup the launch context
        ProgramType programType = ProgramType.valueOfSchedulableType(programInfo.getProgramType());
        ProgramRunner runner = programRunnerFactory.create(programType);
        try {
            if (runner instanceof DistributedProgramRunner) {
                // Call setupLaunchConfig with the corresponding program
                ProgramId programId = program.getId().getParent().program(programType, programInfo.getProgramName());
                ((DistributedProgramRunner) runner).setupLaunchConfig(launchConfig, Programs.create(cConf, program, programId, runner), options, cConf, hConf, tempDir);
                acceptors.add(launchConfig.getClassAcceptor());
            }
        } finally {
            if (runner instanceof Closeable) {
                Closeables.closeQuietly((Closeable) runner);
            }
        }
    }
    // Set the class acceptor
    launchConfig.setClassAcceptor(new AndClassAcceptor(acceptors));
    // Clear and set the runnable for the workflow driver
    launchConfig.clearRunnables();
    Resources resources = findDriverResources(program.getApplicationSpecification().getSpark(), program.getApplicationSpecification().getMapReduce(), spec);
    resources = SystemArguments.getResources(options.getUserArguments(), resources);
    launchConfig.addRunnable(spec.getName(), new WorkflowTwillRunnable(spec.getName()), resources, 1, 0);
}
Also used : WorkflowActionNode(co.cask.cdap.api.workflow.WorkflowActionNode) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) ClassAcceptor(org.apache.twill.api.ClassAcceptor) ProgramId(co.cask.cdap.proto.id.ProgramId) WorkflowSpecification(co.cask.cdap.api.workflow.WorkflowSpecification) SchedulableProgramType(co.cask.cdap.api.schedule.SchedulableProgramType) ProgramType(co.cask.cdap.proto.ProgramType) SchedulableProgramType(co.cask.cdap.api.schedule.SchedulableProgramType) Resources(co.cask.cdap.api.Resources) ScheduleProgramInfo(co.cask.cdap.api.workflow.ScheduleProgramInfo) ProgramRunner(co.cask.cdap.app.runtime.ProgramRunner)

Example 4 with Resources

use of co.cask.cdap.api.Resources in project cdap by caskdata.

the class DistributedServiceProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(LaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) {
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
    // Add a runnable for the service handler
    Resources resources = SystemArguments.getResources(options.getUserArguments(), serviceSpec.getResources());
    launchConfig.addRunnable(serviceSpec.getName(), new ServiceTwillRunnable(serviceSpec.getName()), resources, serviceSpec.getInstances());
}
Also used : ApplicationSpecification(co.cask.cdap.api.app.ApplicationSpecification) ServiceSpecification(co.cask.cdap.api.service.ServiceSpecification) Resources(co.cask.cdap.api.Resources)

Example 5 with Resources

use of co.cask.cdap.api.Resources in project cdap by caskdata.

the class DistributedWebappProgramRunner method setupLaunchConfig.

@Override
protected void setupLaunchConfig(LaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) throws IOException {
    String serviceName = WebappProgramRunner.getServiceName(program.getId());
    Resources resources = SystemArguments.getResources(options.getUserArguments().asMap(), null);
    launchConfig.addRunnable(serviceName, new WebappTwillRunnable(serviceName), resources, 1);
}
Also used : Resources(co.cask.cdap.api.Resources)

Aggregations

Resources (co.cask.cdap.api.Resources)40 JsonObject (com.google.gson.JsonObject)9 Test (org.junit.Test)8 ApplicationSpecification (co.cask.cdap.api.app.ApplicationSpecification)4 Connection (co.cask.cdap.etl.proto.Connection)4 UpgradeContext (co.cask.cdap.etl.proto.UpgradeContext)4 ServiceSpecification (co.cask.cdap.api.service.ServiceSpecification)3 HttpServiceHandlerSpecification (co.cask.cdap.api.service.http.HttpServiceHandlerSpecification)3 ArtifactSelectorConfig (co.cask.cdap.etl.proto.ArtifactSelectorConfig)3 ETLPlugin (co.cask.cdap.etl.proto.v2.ETLPlugin)3 ArrayList (java.util.ArrayList)3 MapReduceSpecification (co.cask.cdap.api.mapreduce.MapReduceSpecification)2 SchedulableProgramType (co.cask.cdap.api.schedule.SchedulableProgramType)2 ServiceHttpEndpoint (co.cask.cdap.api.service.http.ServiceHttpEndpoint)2 SparkSpecification (co.cask.cdap.api.spark.SparkSpecification)2 WorkerSpecification (co.cask.cdap.api.worker.WorkerSpecification)2 ScheduleProgramInfo (co.cask.cdap.api.workflow.ScheduleProgramInfo)2 WorkflowActionNode (co.cask.cdap.api.workflow.WorkflowActionNode)2 BatchPipelineSpec (co.cask.cdap.etl.batch.BatchPipelineSpec)2 Plugin (co.cask.cdap.etl.proto.v1.Plugin)2