Examples with Resources io.cdap.cdap.api.Resources used on opensource projects

Example 6 with Resources

use of io.cdap.cdap.api.Resources in project cdap by caskdata.

the class ServiceSpecificationCodec method deserialize.

@Override
public ServiceSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = (JsonObject) json;
    if (isOldSpec(jsonObj)) {
        return decodeOldSpec(jsonObj);
    }
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    Map<String, HttpServiceHandlerSpecification> handlers = deserializeMap(jsonObj.get("handlers"), context, HttpServiceHandlerSpecification.class);
    Resources resources = context.deserialize(jsonObj.get("resources"), Resources.class);
    int instances = jsonObj.get("instances").getAsInt();
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    return new ServiceSpecification(className, name, description, handlers, resources, instances, plugins, properties);
}

Example 7 with Resources

use of io.cdap.cdap.api.Resources in project cdap by caskdata.

the class MapReduceResourcesTest method testProgrammatic.

@Test
public void testProgrammatic() {
    CConfiguration cConf = CConfiguration.create();
    cConf.setInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB, 300);
    Configuration hConf = new Configuration();
    hConf.setInt(Job.MAP_MEMORY_MB, 3000);
    hConf.setInt(Job.MAP_CPU_VCORES, 5);
    // Always use configurations setup programmatically via job conf.
    MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), null);
    int maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize(3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0);
    validateResources(cConf, hConf, 3000, 5, maxHeapSize);
    // Even resources is provided via context, it is ignored.
    hConf = new Configuration();
    hConf.setInt(Job.MAP_MEMORY_MB, 3000);
    hConf.setInt(Job.MAP_CPU_VCORES, 5);
    MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.emptyMap(), new Resources(1234));
    maxHeapSize = org.apache.twill.internal.utils.Resources.computeMaxHeapSize(3000, cConf.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB), 0);
    validateResources(cConf, hConf, 3000, 5, maxHeapSize);
    // Set the reserved memory via task arguments
    hConf = new Configuration();
    hConf.setInt(Job.MAP_MEMORY_MB, 3000);
    hConf.setInt(Job.MAP_CPU_VCORES, 5);
    MapReduceRuntimeService.TaskType.MAP.configure(hConf, cConf, Collections.singletonMap("system.resources.reserved.memory.override", "2000"), null);
    validateResources(cConf, hConf, 3000, 5, 3000 - 2000);
}

Example 8 with Resources

use of io.cdap.cdap.api.Resources in project cdap by caskdata.

the class SystemArgumentsTest method testSystemResources.

@Test
public void testSystemResources() {
    Resources defaultResources = new Resources();
    // Nothing specified
    Resources resources = SystemArguments.getResources(ImmutableMap.of(), defaultResources);
    Assert.assertEquals(defaultResources, resources);
    // Specify memory
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10"), defaultResources);
    Assert.assertEquals(new Resources(10), resources);
    // Specify cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "8"), defaultResources);
    Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources);
    // Specify both memory and cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "8"), defaultResources);
    Assert.assertEquals(new Resources(10, 8), resources);
    // Specify invalid memory
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-10"), defaultResources);
    Assert.assertEquals(defaultResources, resources);
    // Specify invalid cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.CORES_KEY, "abc"), defaultResources);
    Assert.assertEquals(defaultResources, resources);
    // Specify invalid memory and value cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "xyz", SystemArguments.CORES_KEY, "8"), defaultResources);
    Assert.assertEquals(new Resources(defaultResources.getMemoryMB(), 8), resources);
    // Specify valid memory and invalid cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "10", SystemArguments.CORES_KEY, "-8"), defaultResources);
    Assert.assertEquals(new Resources(10, defaultResources.getVirtualCores()), resources);
    // Specify invalid memory and invalid cores
    resources = SystemArguments.getResources(ImmutableMap.of(SystemArguments.MEMORY_KEY, "-1", SystemArguments.CORES_KEY, "-8"), defaultResources);
    Assert.assertEquals(defaultResources, resources);
    // Specify reserved memory size
    Map<String, String> configs = SystemArguments.getTwillContainerConfigs(ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "200"), 300);
    Assert.assertEquals(ImmutableMap.of(Configs.Keys.JAVA_RESERVED_MEMORY_MB, "200", Configs.Keys.HEAP_RESERVED_MIN_RATIO, "0.33"), configs);
    // Specify invalid reserved memory size
    configs = SystemArguments.getTwillContainerConfigs(ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "-1"), 300);
    Assert.assertTrue(configs.isEmpty());
    // Specify >= container memory size
    configs = SystemArguments.getTwillContainerConfigs(ImmutableMap.of(SystemArguments.RESERVED_MEMORY_KEY_OVERRIDE, "300"), 300);
    Assert.assertTrue(configs.isEmpty());
}

Example 9 with Resources

use of io.cdap.cdap.api.Resources in project cdap by caskdata.

the class MapReduceSpecificationCodec method deserialize.

@Override
public MapReduceSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = json.getAsJsonObject();
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    Resources driverResources = deserializeResources(jsonObj, "driver", context);
    Resources mapperResources = deserializeResources(jsonObj, "mapper", context);
    Resources reducerResources = deserializeResources(jsonObj, "reducer", context);
    JsonElement inputDataSetElem = jsonObj.get("inputDataSet");
    String inputDataSet = inputDataSetElem == null ? null : inputDataSetElem.getAsString();
    JsonElement outputDataSetElem = jsonObj.get("outputDataSet");
    String outputDataSet = outputDataSetElem == null ? null : outputDataSetElem.getAsString();
    Set<String> dataSets = deserializeSet(jsonObj.get("datasets"), context, String.class);
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    return new MapReduceSpecification(className, name, description, inputDataSet, outputDataSet, dataSets, properties, driverResources, mapperResources, reducerResources, plugins);
}

Example 10 with Resources

use of io.cdap.cdap.api.Resources in project cdap by cdapio.

the class ServiceSpecificationCodec method deserialize.

@Override
public ServiceSpecification deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException {
    JsonObject jsonObj = (JsonObject) json;
    if (isOldSpec(jsonObj)) {
        return decodeOldSpec(jsonObj);
    }
    String className = jsonObj.get("className").getAsString();
    String name = jsonObj.get("name").getAsString();
    String description = jsonObj.get("description").getAsString();
    Map<String, Plugin> plugins = deserializeMap(jsonObj.get("plugins"), context, Plugin.class);
    Map<String, HttpServiceHandlerSpecification> handlers = deserializeMap(jsonObj.get("handlers"), context, HttpServiceHandlerSpecification.class);
    Resources resources = context.deserialize(jsonObj.get("resources"), Resources.class);
    int instances = jsonObj.get("instances").getAsInt();
    Map<String, String> properties = deserializeMap(jsonObj.get("properties"), context, String.class);
    return new ServiceSpecification(className, name, description, handlers, resources, instances, plugins, properties);
}