Examples with URI org.gridlab.gat.URI used on opensource projects

Example 1 with URI

use of org.gridlab.gat.URI in project compss by bsc-wdc.

the class GATJob method submit.

@Override
public void submit() throws Exception {
    // Prepare the job
    logger.info("Submit GATJob with ID " + jobId);
    JobDescription jobDescr = null;
    jobDescr = prepareJob();
    // Get a broker for the host
    ResourceBroker broker = null;
    String dest = (String) jobDescr.getResourceDescription().getResourceAttribute(RES_ATTR);
    if ((broker = brokers.get(dest)) == null) {
        broker = GAT.createResourceBroker(context, new URI(dest));
        brokers.put(dest, broker);
    }
    // Submit the job, registering for notifications of job state
    // transitions (associatedJM is the metric listener)
    Job job = null;
    try {
        job = broker.submitJob(jobDescr, this, JOB_STATUS);
        RUNNING_JOBS.add(this);
    } catch (Exception e) {
        if (Tracer.isActivated()) {
            Tracer.freeSlot(((GATWorkerNode) worker.getNode()).getHost(), (Integer) jobDescr.getSoftwareDescription().getAttributes().get("slot"));
        }
        throw e;
    }
    // Update mapping
    GATjob = job;
}

Example 2 with URI

use of org.gridlab.gat.URI in project compss by bsc-wdc.

the class GATWorkerNode method initWorkingDir.

private void initWorkingDir() throws InitNodeException {
    LinkedList<URI> traceScripts = new LinkedList<>();
    LinkedList<String> traceParams = new LinkedList<>();
    String host = getHost();
    String installDir = getInstallDir();
    String workingDir = getWorkingDir();
    String user = getUser();
    if (user == null || user.isEmpty()) {
        user = "";
    } else {
        user += "@";
    }
    try {
        String initScriptPath = Protocol.ANY_URI.getSchema() + user + host + File.separator + installDir + GAT_SCRIPT_PATH + INIT_SCRIPT_NAME;
        traceScripts.add(new URI(initScriptPath));
    } catch (URISyntaxException e) {
        new InitNodeException("Error addind initScript");
    }
    String pars = workingDir;
    traceParams.add(pars);
    // Use cleaner to run the trace script and generate the package
    LOGGER.debug("Initializing working dir " + workingDir + "  in host " + getName());
    boolean result = new GATScriptExecutor(this).executeScript(traceScripts, traceParams, "init_" + host);
    if (!result) {
        throw new InitNodeException("Error executing init script for initializing working dir " + workingDir + " in host " + getName());
    }
}

Example 3 with URI

use of org.gridlab.gat.URI in project compss by bsc-wdc.

the class GATWorkerNode method deleteTemporary.

@Override
public void deleteTemporary() {
    LinkedList<URI> traceScripts = new LinkedList<URI>();
    LinkedList<String> traceParams = new LinkedList<String>();
    String host = getHost();
    String installDir = getInstallDir();
    String workingDir = getWorkingDir();
    String user = getUser();
    if (user == null) {
        user = "";
    } else {
        user += "@";
    }
    try {
        traceScripts.add(new URI(Protocol.ANY_URI.getSchema() + user + host + File.separator + installDir + GAT_SCRIPT_PATH + CLEANER_SCRIPT_NAME));
    } catch (URISyntaxException e) {
        LOGGER.error("Error deleting working dir " + workingDir + " in host " + getName(), e);
        return;
    }
    String pars = workingDir;
    traceParams.add(pars);
    // Use cleaner to run the trace script and generate the package
    LOGGER.debug("Deleting working dir " + workingDir + "  in host " + getName());
    boolean result = new GATScriptExecutor(this).executeScript(traceScripts, traceParams, "clean_" + host);
    if (!result) {
        LOGGER.error("Error executing clean script for deleting working dir " + workingDir + " in host " + getName());
    }
}

Example 4 with URI

use of org.gridlab.gat.URI in project compss by bsc-wdc.

the class GATScriptExecutor method executeScript.

public boolean executeScript(List<URI> scripts, List<String> params, String stdOutFileName) {
    try {
        pool.startThreads();
    } catch (Exception e) {
        logger.error(THREAD_POOL_START_ERR, e);
        return false;
    }
    synchronized (jobQueue) {
        jobCount = scripts.size();
    }
    for (int i = 0; i < scripts.size(); i++) {
        URI script = scripts.get(i);
        String cleanParam = params.get(i);
        if (script == null) {
            continue;
        }
        if (debug) {
            logger.debug("Clean call: " + script + " " + cleanParam);
        }
        try {
            if (!node.isUserNeeded() && script.getUserInfo() != null) {
                // Remove user from the URI
                script.setUserInfo(null);
            }
            String user = script.getUserInfo();
            if (user == null) {
                user = "";
            } else {
                user += "@";
            }
            SoftwareDescription sd = new SoftwareDescription();
            sd.addAttribute("uri", Protocol.ANY_URI.getSchema() + user + script.getHost());
            sd.setExecutable(script.getPath());
            sd.setArguments(cleanParam.split(" "));
            sd.addAttribute("job_number", i);
            sd.addAttribute(SoftwareDescription.SANDBOX_ROOT, File.separator + "tmp" + File.separator);
            sd.addAttribute(SoftwareDescription.SANDBOX_USEROOT, "true");
            sd.addAttribute(SoftwareDescription.SANDBOX_DELETE, "false");
            if (debug) {
                try {
                    org.gridlab.gat.io.File outFile = GAT.createFile(node.getContext(), Protocol.ANY_URI.getSchema() + File.separator + System.getProperty(COMPSsConstants.APP_LOG_DIR) + File.separator + stdOutFileName + ".out");
                    sd.setStdout(outFile);
                    org.gridlab.gat.io.File errFile = GAT.createFile(node.getContext(), Protocol.ANY_URI.getSchema() + File.separator + System.getProperty(COMPSsConstants.APP_LOG_DIR) + File.separator + stdOutFileName + ".err");
                    sd.setStderr(errFile);
                } catch (Exception e) {
                    logger.error(CLEAN_JOB_ERR, e);
                }
            }
            sdQueue.enqueue(sd);
        } catch (Exception e) {
            logger.error(CLEAN_JOB_ERR, e);
            return false;
        }
    }
    Long timeout = System.currentTimeMillis() + 60_000l;
    // Poll for completion of the clean jobs
    while (jobCount > 0 && System.currentTimeMillis() < timeout) {
        Job job = jobQueue.dequeue();
        if (job == null) {
            synchronized (jobQueue) {
                jobCount--;
            }
        } else if (job.getState() == JobState.STOPPED) {
            synchronized (jobQueue) {
                jobCount--;
            }
        } else if (job.getState() == JobState.SUBMISSION_ERROR) {
            logger.error(CLEAN_JOB_ERR + ": " + job);
            synchronized (jobQueue) {
                jobCount--;
            }
        } else {
            jobQueue.enqueue(job);
            try {
                Thread.sleep(50);
            } catch (Exception e) {
            }
        }
    }
    try {
        pool.stopThreads();
    } catch (Exception e) {
        logger.error(THREAD_POOL_STOP_ERR, e);
        return false;
    }
    // Move cleanX.out logs to default logger
    if (debug) {
        String stdOutFilePath = System.getProperty(COMPSsConstants.APP_LOG_DIR) + File.separator + stdOutFileName + ".out";
        try (FileReader cleanOut = new FileReader(stdOutFilePath);
            BufferedReader br = new BufferedReader(cleanOut)) {
            String line = br.readLine();
            while (line != null) {
                logger.debug(line);
                line = br.readLine();
            }
        } catch (Exception e) {
            logger.error("Error moving std out file", e);
        }
        // Delete file
        if (!new File(stdOutFilePath).delete()) {
            logger.error("Error deleting out file " + stdOutFilePath);
        }
    }
    // Move cleanX.err logs to default logger
    if (debug) {
        String stdErrFilePath = System.getProperty(COMPSsConstants.APP_LOG_DIR) + File.separator + stdOutFileName + ".err";
        try (FileReader cleanErr = new FileReader(stdErrFilePath);
            BufferedReader br = new BufferedReader(cleanErr)) {
            String line = br.readLine();
            while (line != null) {
                logger.error(line);
                line = br.readLine();
            }
        } catch (Exception e) {
            logger.error("Error moving std err file", e);
        }
        if (!new File(stdErrFilePath).delete()) {
            logger.error("Error deleting err file " + stdErrFilePath);
        }
    }
    return true;
}

Example 5 with URI

use of org.gridlab.gat.URI in project compss by bsc-wdc.

the class GATCopy method specificCopy.

@Override
public void specificCopy() throws CopyException {
    LOGGER.debug(DBG_PREFIX + "Performing GAT Specific Copy for " + getName());
    // Fetch valid destination URIs
    List<MultiURI> targetURIs = tgtLoc.getURIs();
    List<URI> selectedTargetURIs = new LinkedList<>();
    for (MultiURI uri : targetURIs) {
        try {
            URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
            if (internalURI != null) {
                selectedTargetURIs.add(internalURI);
            }
        } catch (UnstartedNodeException une) {
            throw new GATCopyException(une);
        }
    }
    if (selectedTargetURIs.isEmpty()) {
        LOGGER.error(DBG_PREFIX + ERR_NO_TGT_URI);
        throw new GATCopyException(ERR_NO_TGT_URI);
    }
    LOGGER.debug(DBG_PREFIX + "Selected target URIs");
    // Fetch valid source URIs
    List<MultiURI> sourceURIs;
    List<URI> selectedSourceURIs = new LinkedList<>();
    synchronized (srcData) {
        if (srcLoc != null) {
            sourceURIs = srcLoc.getURIs();
            for (MultiURI uri : sourceURIs) {
                try {
                    URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                    if (internalURI != null) {
                        selectedSourceURIs.add(internalURI);
                    }
                } catch (UnstartedNodeException une) {
                    LOGGER.error(DBG_PREFIX + "Exception selecting source URI");
                    throw new GATCopyException(une);
                }
            }
        }
        sourceURIs = srcData.getURIs();
        for (MultiURI uri : sourceURIs) {
            try {
                URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                if (internalURI != null) {
                    selectedSourceURIs.add(internalURI);
                }
            } catch (UnstartedNodeException une) {
                LOGGER.error(DBG_PREFIX + "Exception selecting source URI for " + getName());
                throw new GATCopyException(une);
            }
        }
        if (selectedSourceURIs.isEmpty()) {
            if (srcData.isInMemory()) {
                LOGGER.debug("Data for " + getName() + " is in memory");
                try {
                    srcData.writeToStorage();
                    sourceURIs = srcData.getURIs();
                    for (MultiURI uri : sourceURIs) {
                        URI internalURI = (URI) uri.getInternalURI(GATAdaptor.ID);
                        if (internalURI != null) {
                            selectedSourceURIs.add(internalURI);
                        }
                    }
                } catch (Exception e) {
                    LOGGER.fatal("Exception writing object to file.", e);
                    throw new GATCopyException(ERR_NO_SRC_URI);
                }
            } else {
                LOGGER.error(DBG_PREFIX + ERR_NO_SRC_URI);
                throw new GATCopyException(ERR_NO_SRC_URI);
            }
        }
    }
    GATInvocationException exception = new GATInvocationException("default logical file");
    for (URI src : selectedSourceURIs) {
        for (URI tgt : selectedTargetURIs) {
            try {
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("GATCopy From: " + src + " to " + tgt);
                }
                // Source and target URIs contain Runtime information (schema)
                // Convert it to GAT format
                URI gatSrc = new URI(DataLocation.Protocol.ANY_URI.getSchema() + src.getHost() + "/" + src.getPath());
                URI gatTgt = new URI(DataLocation.Protocol.ANY_URI.getSchema() + tgt.getHost() + "/" + tgt.getPath());
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("GATCopy From: " + gatSrc + " to " + gatTgt);
                }
                doCopy(gatSrc, gatTgt);
            // Try to copy from each location until successful
            } catch (Exception e) {
                exception.add("default logical file", e);
                LOGGER.warn("Error copying file", e);
                continue;
            }
            return;
        }
    }
    if (!(this.reason instanceof WorkersDebugInfoCopyTransferable)) {
        ErrorManager.error("File '" + srcData.getName() + "' could not be copied because it does not exist.", exception);
    }
    throw new GATCopyException(exception);
}