Examples with MethodConfiguration es.bsc.compss.types.resources.configuration.MethodConfiguration used on opensource projects

Example 1 with MethodConfiguration

use of es.bsc.compss.types.resources.configuration.MethodConfiguration in project compss by bsc-wdc.

the class ResourceLoader method loadComputeNode.

private static boolean loadComputeNode(ComputeNodeType cn_project, es.bsc.compss.types.resources.jaxb.ComputeNodeType cn_resources) {
    // Add the name
    String name = cn_project.getName();
    /* Add properties given by the resources file **************************************** */
    MethodResourceDescription mrd = new MethodResourceDescription();
    List<es.bsc.compss.types.resources.jaxb.ProcessorType> processors = resources.getProcessors(cn_resources);
    if (processors != null) {
        for (es.bsc.compss.types.resources.jaxb.ProcessorType p : processors) {
            String procName = p.getName();
            int computingUnits = resources.getProcessorComputingUnits(p);
            String architecture = resources.getProcessorArchitecture(p);
            float speed = resources.getProcessorSpeed(p);
            String type = resources.getProcessorType(p);
            float internalMemory = resources.getProcessorMemorySize(p);
            ProcessorPropertyType procProp = resources.getProcessorProperty(p);
            String propKey = (procProp != null) ? procProp.getKey() : "";
            String propValue = (procProp != null) ? procProp.getValue() : "";
            mrd.addProcessor(procName, computingUnits, architecture, speed, type, internalMemory, propKey, propValue);
        }
    }
    mrd.setMemorySize(resources.getMemorySize(cn_resources));
    mrd.setMemoryType(resources.getMemoryType(cn_resources));
    mrd.setStorageSize(resources.getStorageSize(cn_resources));
    mrd.setStorageType(resources.getStorageType(cn_resources));
    mrd.setOperatingSystemType(resources.getOperatingSystemType(cn_resources));
    mrd.setOperatingSystemDistribution(resources.getOperatingSystemDistribution(cn_resources));
    mrd.setOperatingSystemVersion(resources.getOperatingSystemVersion(cn_resources));
    List<String> apps = resources.getApplications(cn_resources);
    if (apps != null) {
        for (String appName : apps) {
            mrd.addApplication(appName);
        }
    }
    es.bsc.compss.types.resources.jaxb.PriceType p = resources.getPrice(cn_resources);
    if (p != null) {
        mrd.setPriceTimeUnit(p.getTimeUnit());
        mrd.setPricePerUnit(p.getPricePerUnit());
    }
    // Add Shared Disks (Name, mountpoint)
    Map<String, String> sharedDisks = resources.getSharedDisks(cn_resources);
    if (sharedDisks != null) {
        List<String> declaredSharedDisks = resources.getSharedDisks_names();
        for (String diskName : sharedDisks.keySet()) {
            if (declaredSharedDisks == null || !declaredSharedDisks.contains(diskName)) {
                ErrorManager.warn("SharedDisk " + diskName + " defined in the ComputeNode " + name + " is not defined in the resources.xml. Skipping");
                sharedDisks.remove(diskName);
            // TODO: Check the disk information (size and type)
            }
        }
    }
    // Add the adaptors properties (queue types and adaptor properties)
    // TODO Support multiple adaptor properties
    String loadedAdaptor = System.getProperty(COMPSsConstants.COMM_ADAPTOR);
    List<String> queues_project = project.getAdaptorQueues(cn_project, loadedAdaptor);
    List<String> queues_resources = resources.getAdaptorQueues(cn_resources, loadedAdaptor);
    if (queues_project == null) {
        // Has no tag adaptors on project, get default resources complete
        for (String queue : queues_resources) {
            mrd.addHostQueue(queue);
        }
    } else {
        // Project defines a subset of queues
        for (String queue : queues_resources) {
            if (queues_project.contains(queue)) {
                mrd.addHostQueue(queue);
            }
        }
    }
    Object adaptorProperties_project = project.getAdaptorProperties(cn_project, loadedAdaptor);
    Object adaptorProperties_resources = resources.getAdaptorProperties(cn_resources, loadedAdaptor);
    MethodConfiguration config = null;
    try {
        config = (MethodConfiguration) Comm.constructConfiguration(loadedAdaptor, adaptorProperties_project, adaptorProperties_resources);
    } catch (ConstructConfigurationException cce) {
        ErrorManager.warn("Adaptor " + loadedAdaptor + " configuration constructor failed", cce);
        return false;
    }
    // If we have reached this point the config is SURELY not null
    /* Add properties given by the project file **************************************** */
    config.setHost(cn_project.getName());
    config.setUser(project.getUser(cn_project));
    config.setInstallDir(project.getInstallDir(cn_project));
    config.setWorkingDir(project.getWorkingDir(cn_project));
    int limitOfTasks = project.getLimitOfTasks(cn_project);
    if (limitOfTasks >= 0) {
        config.setLimitOfTasks(limitOfTasks);
    } else {
        config.setLimitOfTasks(mrd.getTotalCPUComputingUnits());
    }
    config.setTotalComputingUnits(mrd.getTotalCPUComputingUnits());
    config.setTotalGPUComputingUnits(mrd.getTotalGPUComputingUnits());
    config.setTotalFPGAComputingUnits(mrd.getTotalFPGAComputingUnits());
    config.setTotalOTHERComputingUnits(mrd.getTotalOTHERComputingUnits());
    ApplicationType app = project.getApplication(cn_project);
    if (app != null) {
        config.setAppDir(app.getAppDir());
        config.setLibraryPath(app.getLibraryPath());
        config.setClasspath(app.getClasspath());
        config.setPythonpath(app.getPythonpath());
    }
    /* Pass all the information to the ResourceManager to insert it into the Runtime ** */
    LOGGER.debug("Adding method worker " + name);
    MethodWorker methodWorker = createMethodWorker(name, mrd, sharedDisks, config);
    ResourceManager.addStaticResource(methodWorker);
    // If we have reached this point the method worker has been correctly created
    return true;
}

Example 2 with MethodConfiguration

use of es.bsc.compss.types.resources.configuration.MethodConfiguration in project compss by bsc-wdc.

the class Converter method getSoftwareDescription.

/**
 * Returns the software description
 *
 * @param cmrd
 * @return
 */
public static SoftwareDescription getSoftwareDescription(CloudMethodResourceDescription cmrd) {
    String osType = cmrd.getOperatingSystemType();
    String osDist = cmrd.getOperatingSystemDistribution();
    String osVersion = cmrd.getOperatingSystemVersion();
    List<String> apps = cmrd.getAppSoftware();
    CloudImageDescription cid = cmrd.getImage();
    MethodConfiguration mc = cid.getConfig();
    InstallationDescription installDesc = getInstallationDescription(mc);
    return new SoftwareDescription(osType, osDist, osVersion, apps, installDesc);
}

Example 3 with MethodConfiguration

use of es.bsc.compss.types.resources.configuration.MethodConfiguration in project compss by bsc-wdc.

the class ResourceManagerTest method testStaticWorkersOperations.

@Test
public void testStaticWorkersOperations() {
    // Add Method Resource 1
    MethodResourceDescription mrd1 = new MethodResourceDescription();
    Float memory1 = 10f + (float) Math.random();
    mrd1.setMemorySize(memory1);
    MethodConfiguration mc1 = null;
    try {
        mc1 = (MethodConfiguration) Comm.constructConfiguration(FakeMethodAdaptor.class.getName(), null, null);
    } catch (Exception e) {
    }
    String worker1Name = "Worker" + (int) (Math.random() * 10000);
    MethodWorker mw1 = createMethodWorker(worker1Name, mrd1, new HashMap<>(), mc1);
    ResourceManager.addStaticResource(mw1);
    if (ResourceManager.getTotalNumberOfWorkers() != 1) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getStaticResources().contains(mw1)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(worker1Name) != mw1) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getWorker(worker1Name).getName().equals(worker1Name)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (((MethodResourceDescription) ResourceManager.getWorker(worker1Name).getDescription()).getMemorySize() != mrd1.getMemorySize()) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    // Add Method Resource 2
    MethodResourceDescription mrd2 = new MethodResourceDescription();
    Float memory2 = 10f + (float) Math.random();
    mrd2.setMemorySize(memory2);
    MethodConfiguration mc2 = null;
    try {
        mc2 = (MethodConfiguration) Comm.constructConfiguration(FakeMethodAdaptor.class.getName(), null, null);
    } catch (Exception e) {
    }
    String worker2Name = "Worker" + (int) (Math.random() * 10000);
    MethodWorker mw2 = createMethodWorker(worker2Name, mrd2, new HashMap<>(), mc2);
    ResourceManager.addStaticResource(mw2);
    if (ResourceManager.getTotalNumberOfWorkers() != 2) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getStaticResources().contains(mw2)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(worker2Name) != mw2) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getWorker(worker2Name).getName().equals(worker2Name)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (((MethodResourceDescription) ResourceManager.getWorker(worker2Name).getDescription()).getMemorySize() != mrd2.getMemorySize()) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    // Add Service Resource 1
    ServiceResourceDescription srd1 = new ServiceResourceDescription("service1", "namespace1", "port1", 2);
    ServiceConfiguration sc1 = null;
    try {
        sc1 = (ServiceConfiguration) Comm.constructConfiguration(FakeServiceAdaptor.class.getName(), null, null);
    } catch (Exception e) {
    }
    String wsdl1 = "WSDL" + (int) (Math.random() * 10000);
    ServiceWorker sw1 = createServiceWorker(wsdl1, srd1, sc1);
    ResourceManager.addStaticResource(sw1);
    if (ResourceManager.getTotalNumberOfWorkers() != 3) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getStaticResources().contains(sw1)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(wsdl1) != sw1) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!ResourceManager.getWorker(wsdl1).getName().equals(wsdl1)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (!((ServiceResourceDescription) ResourceManager.getWorker(wsdl1).getDescription()).getNamespace().equals(srd1.getNamespace())) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    ResourceManager.removeWorker(sw1);
    if (ResourceManager.getTotalNumberOfWorkers() != 2) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getStaticResources().contains(sw1)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(wsdl1) != null) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    ResourceManager.removeWorker(mw1);
    if (ResourceManager.getTotalNumberOfWorkers() != 1) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getStaticResources().contains(worker1Name)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(worker1Name) != null) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    ResourceManager.removeWorker(mw2);
    if (ResourceManager.getTotalNumberOfWorkers() != 0) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getStaticResources().contains(worker2Name)) {
        fail("ResourceManager is not properly adding Method Workers");
    }
    if (ResourceManager.getWorker(worker2Name) != null) {
        fail("ResourceManager is not properly adding Method Workers");
    }
}

Example 4 with MethodConfiguration

use of es.bsc.compss.types.resources.configuration.MethodConfiguration in project compss by bsc-wdc.

the class ResourceLoader method createImage.

private static CloudImageDescription createImage(ImageType im_project, es.bsc.compss.types.resources.jaxb.ImageType im_resources, Map<String, String> properties) {
    String imageName = im_project.getName();
    LOGGER.debug("Loading Image" + imageName);
    CloudImageDescription cid = new CloudImageDescription(imageName, properties);
    /* Add properties given by the resources file **************************************** */
    cid.setOperatingSystemType(resources.getOperatingSystemType(im_resources));
    cid.setOperatingSystemDistribution(resources.getOperatingSystemDistribution(im_resources));
    cid.setOperatingSystemVersion(resources.getOperatingSystemVersion(im_resources));
    List<String> apps = resources.getApplications(im_resources);
    if (apps != null) {
        for (String appName : apps) {
            cid.addApplication(appName);
        }
    }
    es.bsc.compss.types.resources.jaxb.PriceType p = resources.getPrice(im_resources);
    if (p != null) {
        cid.setPriceTimeUnit(p.getTimeUnit());
        cid.setPricePerUnit(p.getPricePerUnit());
    }
    cid.setCreationTime(resources.getCreationTime(im_resources));
    // Add Shared Disks (Name, mountpoint)
    Map<String, String> sharedDisks = resources.getSharedDisks(im_resources);
    if (sharedDisks != null) {
        List<String> declaredSharedDisks = resources.getSharedDisks_names();
        for (String diskName : sharedDisks.keySet()) {
            if (declaredSharedDisks == null || !declaredSharedDisks.contains(diskName)) {
                ErrorManager.warn("SharedDisk " + diskName + " defined in the Image " + imageName + " is not defined in the resources.xml. Skipping");
                sharedDisks.remove(diskName);
            // TODO: Check the disk information (size and type)
            }
        }
    }
    cid.setSharedDisks(sharedDisks);
    // Add the adaptors properties (queue types and adaptor properties)
    // TODO Support multiple adaptor properties
    String loadedAdaptor = System.getProperty(COMPSsConstants.COMM_ADAPTOR);
    List<String> queues_project = project.getAdaptorQueues(im_project, loadedAdaptor);
    List<String> queues_resources = resources.getAdaptorQueues(im_resources, loadedAdaptor);
    for (String queue : queues_resources) {
        if (queues_project.contains(queue)) {
            cid.addQueue(queue);
        }
    }
    Object adaptorProperties_project = project.getAdaptorProperties(im_project, loadedAdaptor);
    Object adaptorProperties_resources = resources.getAdaptorProperties(im_resources, loadedAdaptor);
    MethodConfiguration config = null;
    try {
        config = (MethodConfiguration) Comm.constructConfiguration(loadedAdaptor, adaptorProperties_project, adaptorProperties_resources);
    } catch (ConstructConfigurationException cce) {
        ErrorManager.warn("Adaptor configuration constructor failed", cce);
        return null;
    }
    // If we have reached this point the mp is SURELY not null
    /* Add properties given by the project file **************************************** */
    config.setHost(im_project.getName());
    config.setUser(project.getUser(im_project));
    config.setInstallDir(project.getInstallDir(im_project));
    config.setWorkingDir(project.getWorkingDir(im_project));
    config.setLimitOfTasks(project.getLimitOfTasks(im_project));
    ApplicationType app = project.getApplication(im_project);
    if (app != null) {
        config.setAppDir(app.getAppDir());
        config.setLibraryPath(app.getLibraryPath());
        config.setClasspath(app.getClasspath());
        config.setPythonpath(app.getClasspath());
    }
    List<PackageType> packages = project.getPackages(im_project);
    for (PackageType pack : packages) {
        cid.addPackage(pack.getSource(), pack.getTarget());
        SoftwareListType software = pack.getIncludedSoftware();
        if (software != null) {
            cid.addAllApplications(software.getApplication());
        }
    }
    // Add configuration to image
    cid.setConfig(config);
    return cid;
}

Example 5 with MethodConfiguration

use of es.bsc.compss.types.resources.configuration.MethodConfiguration in project compss by bsc-wdc.

the class CreationThread method prepareNewResource.

private CloudMethodWorker prepareNewResource(VM vm) throws ConnectorException {
    CloudMethodResourceDescription granted = vm.getDescription();
    CloudImageDescription cid = granted.getImage();
    Map<String, String> workerProperties = cid.getProperties();
    String user = cid.getConfig().getUser();
    String password = workerProperties.get(AbstractConnector.PROPERTY_PASSW_NAME);
    try {
        operations.configureAccess(granted.getName(), user, password);
    } catch (ConnectorException e) {
        RUNTIME_LOGGER.error(ERROR_CONFIGURE_ACCESS_VM + granted.getName(), e);
        RESOURCE_LOGGER.error("ERROR_MSG = [\n\t" + ERROR_CONFIGURE_ACCESS_VM + "\n\tNAME = " + granted.getName() + "\n\tPROVIDER =  " + provider + "\n]", e);
        throw e;
    }
    try {
        operations.prepareMachine(granted.getName(), cid);
    } catch (ConnectorException e) {
        RUNTIME_LOGGER.error(ERROR_PREPARING_VM + granted.getName(), e);
        RESOURCE_LOGGER.error("ERROR_MSG = [\n\t" + ERROR_PREPARING_VM + granted.getName() + "]", e);
        throw e;
    }
    CloudMethodWorker worker;
    MethodConfiguration mc = cid.getConfig();
    int limitOfTasks = mc.getLimitOfTasks();
    int computingUnits = granted.getTotalCPUComputingUnits();
    if (limitOfTasks < 0 && computingUnits < 0) {
        mc.setLimitOfTasks(0);
        mc.setTotalComputingUnits(0);
    } else {
        mc.setLimitOfTasks(Math.max(limitOfTasks, computingUnits));
        mc.setTotalComputingUnits(Math.max(limitOfTasks, computingUnits));
    }
    mc.setHost(granted.getName());
    mc.setLimitOfGPUTasks(granted.getTotalGPUComputingUnits());
    mc.setTotalGPUComputingUnits(granted.getTotalGPUComputingUnits());
    mc.setLimitOfFPGATasks(granted.getTotalFPGAComputingUnits());
    mc.setTotalFPGAComputingUnits(granted.getTotalFPGAComputingUnits());
    mc.setLimitOfOTHERSTasks(granted.getTotalOTHERComputingUnits());
    mc.setTotalOTHERComputingUnits(granted.getTotalOTHERComputingUnits());
    worker = new CloudMethodWorker(granted.getName(), provider, granted, mc, cid.getSharedDisks());
    try {
        worker.announceCreation();
    } catch (Exception e) {
        RUNTIME_LOGGER.error("Machine " + granted.getName() + " shut down because an error announcing creation");
        RESOURCE_LOGGER.error("ERROR_MSG = [\n\t" + ERROR_ANNOUNCE_VM + "\n\tNAME = " + granted.getName() + "\n\tPROVIDER =  " + provider + "\n]", e);
        throw new ConnectorException(e);
    }
    // Add the new machine to ResourceManager
    if (operations.getTerminate()) {
        RESOURCE_LOGGER.info("INFO_MSG = [\n\t" + WARN_VM_REFUSED + "\n\tRESOURCE_NAME = " + granted.getName() + "\n]");
        try {
            worker.announceDestruction();
        } catch (Exception e) {
            RESOURCE_LOGGER.error("ERROR_MSG = [\n\t" + ERROR_ANNOUNCE_VM_DESTROY + "\n\tVM_NAME = " + granted.getName() + "\n]", e);
        }
        Semaphore sem = new Semaphore(0);
        ShutdownListener sl = new ShutdownListener(sem);
        worker.stop(sl);
        sl.enable();
        try {
            sem.acquire();
        } catch (Exception e) {
            RESOURCE_LOGGER.error(ERROR_WORKER_SHUTDOWN);
        }
        throw new ConnectorException(ERROR_USELESS_VM);
    }
    return worker;
}