Examples with NIONode es.bsc.comm.nio.NIONode used on opensource projects

Example 1 with NIONode

use of es.bsc.comm.nio.NIONode in project compss by bsc-wdc.

the class WorkerStarter method startWorker.

/**
 * Starts the current worker
 *
 * @return
 * @throws InitNodeException
 */
public NIONode startWorker() throws InitNodeException {
    String name = this.nw.getName();
    String user = this.nw.getUser();
    int minPort = this.nw.getConfiguration().getMinPort();
    int maxPort = this.nw.getConfiguration().getMaxPort();
    int port = minPort;
    // Solves exit error 143
    synchronized (addressToWorkerStarter) {
        addressToWorkerStarter.put(name, this);
        LOGGER.debug("[WorkerStarter] Worker starter for " + name + " registers in the hashmap");
    }
    NIONode n = null;
    int pid = -1;
    while (port <= maxPort && !this.toStop) {
        // Kill previous worker processes if any
        killPreviousWorker(user, name, pid);
        // Instantiate the node
        n = new NIONode(name, port);
        // Start the worker
        pid = startWorker(user, name, port);
        // Check worker status
        LOGGER.info("[WorkerStarter] Worker process started. Checking connectivity...");
        checkWorker(n, name);
        // Check received ack
        LOGGER.debug("[WorkerStarter] Retries for " + name + " have finished.");
        if (!this.workerIsReady) {
            // Try next port
            ++port;
        } else {
            // Success, return node
            try {
                Runtime.getRuntime().addShutdownHook(new Ender(this, this.nw, pid));
            } catch (IllegalStateException e) {
                LOGGER.warn("Tried to shutdown vm while it was already being shutdown", e);
            }
            return n;
        }
    }
    // This can be because node is stopping or because we reached the maximum available ports
    if (this.toStop) {
        String msg = "[STOP]: Worker " + name + " stopped during creation because application is stopped";
        LOGGER.warn(msg);
        throw new InitNodeException(msg);
    } else if (!this.workerIsReady) {
        String msg = "[TIMEOUT]: Could not start the NIO worker on resource " + name + " through user " + user + ".";
        LOGGER.warn(msg);
        throw new InitNodeException(msg);
    } else {
        String msg = "[UNKNOWN]: Could not start the NIO worker on resource " + name + " through user " + user + ".";
        LOGGER.warn(msg);
        throw new InitNodeException(msg);
    }
}

Example 2 with NIONode

use of es.bsc.comm.nio.NIONode in project compss by bsc-wdc.

the class NIOWorker method main.

public static void main(String[] args) {
    // Check arguments length
    if (args.length != (NUM_PARAMS_NIO_WORKER)) {
        if (WORKER_LOGGER_DEBUG) {
            WORKER_LOGGER.debug("Received parameters: ");
            for (int i = 0; i < args.length; ++i) {
                WORKER_LOGGER.debug("Param " + i + ":  " + args[i]);
            }
        }
        ErrorManager.fatal(ERROR_INCORRECT_NUM_PARAMS);
    }
    // Parse arguments
    isWorkerDebugEnabled = Boolean.valueOf(args[0]);
    int maxSnd = Integer.parseInt(args[1]);
    int maxRcv = Integer.parseInt(args[2]);
    String workerIP = args[3];
    int wPort = Integer.parseInt(args[4]);
    int mPort = Integer.parseInt(args[5]);
    int computingUnitsCPU = Integer.parseInt(args[6]);
    int computingUnitsGPU = Integer.parseInt(args[7]);
    String cpuMap = args[8];
    String gpuMap = args[9];
    int limitOfTasks = Integer.parseInt(args[10]);
    String appUuid = args[11];
    String lang = args[12];
    String workingDir = args[13];
    String installDir = args[14];
    String appDir = args[15];
    String libPath = args[16];
    String classpath = args[17];
    String pythonpath = args[18];
    String trace = args[19];
    String extraeFile = args[20];
    String host = args[21];
    storageConf = args[22];
    executionType = args[23];
    persistentC = Boolean.parseBoolean(args[24]);
    // Print arguments
    if (isWorkerDebugEnabled) {
        WORKER_LOGGER.debug("maxSnd: " + String.valueOf(maxSnd));
        WORKER_LOGGER.debug("maxRcv: " + String.valueOf(maxRcv));
        WORKER_LOGGER.debug("WorkerName: " + workerIP);
        WORKER_LOGGER.debug("WorkerPort: " + String.valueOf(wPort));
        WORKER_LOGGER.debug("MasterPort: " + String.valueOf(mPort));
        WORKER_LOGGER.debug("Computing Units CPU: " + String.valueOf(computingUnitsCPU));
        WORKER_LOGGER.debug("Computing Units GPU: " + String.valueOf(computingUnitsGPU));
        WORKER_LOGGER.debug("User defined CPU Map: " + cpuMap);
        WORKER_LOGGER.debug("User defined GPU Map: " + gpuMap);
        WORKER_LOGGER.debug("Limit Of Tasks: " + String.valueOf(limitOfTasks));
        WORKER_LOGGER.debug("App uuid: " + appUuid);
        WORKER_LOGGER.debug("WorkingDir:" + workingDir);
        WORKER_LOGGER.debug("Install Dir: " + installDir);
        WORKER_LOGGER.debug("Tracing: " + trace);
        WORKER_LOGGER.debug("Extrae config File: " + extraeFile);
        WORKER_LOGGER.debug("Host: " + host);
        WORKER_LOGGER.debug("LibraryPath: " + libPath);
        WORKER_LOGGER.debug("Classpath: " + classpath);
        WORKER_LOGGER.debug("Pythonpath: " + pythonpath);
        WORKER_LOGGER.debug("StorageConf: " + storageConf);
        WORKER_LOGGER.debug("executionType: " + executionType);
        WORKER_LOGGER.debug("Persistent c: " + persistentC);
        WORKER_LOGGER.debug("Remove Sanbox WD: " + removeWD);
    }
    // Configure storage
    System.setProperty(COMPSsConstants.STORAGE_CONF, storageConf);
    try {
        if (storageConf == null || storageConf.equals("") || storageConf.equals("null")) {
            WORKER_LOGGER.warn("No storage configuration file passed");
        } else {
            StorageItf.init(storageConf);
        }
    } catch (StorageException e) {
        ErrorManager.fatal("Error loading storage configuration file: " + storageConf, e);
    }
    // Configure tracing
    System.setProperty(COMPSsConstants.EXTRAE_CONFIG_FILE, extraeFile);
    tracing_level = Integer.parseInt(trace);
    if (tracing_level > 0) {
        NIOTracer.init(tracing_level);
        NIOTracer.emitEvent(NIOTracer.Event.START.getId(), NIOTracer.Event.START.getType());
        try {
            tracingID = Integer.parseInt(host);
            NIOTracer.setWorkerInfo(installDir, workerIP, workingDir, tracingID);
        } catch (Exception e) {
            WORKER_LOGGER.error("No valid hostID provided to the tracing system. Provided ID: " + host);
        }
    }
    /*
         * ***********************************************************************************************************
         * LAUNCH THE WORKER
         *************************************************************************************************************/
    NIOWorker nw = new NIOWorker(maxSnd, maxRcv, workerIP, mPort, computingUnitsCPU, computingUnitsGPU, cpuMap, gpuMap, limitOfTasks, appUuid, lang, workingDir, installDir, appDir, libPath, classpath, pythonpath);
    NIOMessageHandler mh = new NIOMessageHandler(nw);
    // Initialize the Transfer Manager
    WORKER_LOGGER.debug("  Initializing the TransferManager structures...");
    try {
        TM.init(NIO_EVENT_MANAGER_CLASS, null, mh);
    } catch (CommException ce) {
        WORKER_LOGGER.error("Error initializing Transfer Manager on worker " + nw.getHostName(), ce);
        // Shutdown the Worker since the error it is not recoverable
        nw.shutdown(null);
        return;
    }
    // Start the Transfer Manager thread (starts the EventManager)
    WORKER_LOGGER.debug("  Starting TransferManager Thread");
    TM.start();
    try {
        TM.startServer(new NIONode(null, wPort));
    } catch (CommException ce) {
        WORKER_LOGGER.error("Error starting TransferManager Server at Worker" + nw.getHostName(), ce);
        nw.shutdown(null);
        return;
    }
    if (NIOTracer.isActivated()) {
        NIOTracer.emitEvent(NIOTracer.EVENT_END, NIOTracer.Event.START.getType());
    }
    // Wait for the Transfer Manager thread to finish (the shutdown is received on that thread)
    try {
        TM.join();
    } catch (InterruptedException ie) {
        WORKER_LOGGER.warn("TransferManager interrupted", ie);
        Thread.currentThread().interrupt();
    }
}

Example 3 with NIONode

use of es.bsc.comm.nio.NIONode in project compss by bsc-wdc.

the class NIOAgent method requestTransfers.

/**
 * Check if receive slots available
 */
public void requestTransfers() {
    DataRequest dr = null;
    synchronized (pendingRequests) {
        if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
            dr = pendingRequests.remove();
        }
    }
    while (dr != null) {
        Data source = dr.getSource();
        NIOURI uri = source.getFirstURI();
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(source.getName());
        }
        NIONode nn = uri.getHost();
        if (nn.getIp() == null) {
            nn = masterNode;
        }
        Connection c = null;
        try {
            c = TM.startConnection(nn);
            LOGGER.debug("Connection " + c.hashCode() + " will be used to acquire data " + dr.getTarget() + " stored in " + nn + " with name " + dr.getSource().getName());
            Data remoteData = new Data(source.getName(), uri);
            CommandDataDemand cdd = new CommandDataDemand(this, remoteData, tracingID);
            ongoingTransfers.put(c, dr.getSource().getName());
            c.sendCommand(cdd);
            if (NIOTracer.isActivated()) {
                c.receive();
            }
            if (dr.getType() == DataType.FILE_T) {
                c.receiveDataFile(dr.getTarget());
            } else {
                c.receiveDataObject();
            }
        } catch (Exception e) {
            e.printStackTrace(System.err);
        } finally {
            if (c != null) {
                c.finishConnection();
            }
        }
        synchronized (pendingRequests) {
            if (!pendingRequests.isEmpty() && tryAcquireReceiveSlot()) {
                dr = pendingRequests.remove();
            } else {
                dr = null;
            }
        }
        if (NIOTracer.isActivated()) {
            NIOTracer.emitDataTransferEvent(NIOTracer.TRANSFER_END);
        }
    }
}

Example 4 with NIONode

use of es.bsc.comm.nio.NIONode in project compss by bsc-wdc.

the class NIOWorkerNode method start.

@Override
public void start() throws InitNodeException {
    NIONode n = null;
    try {
        this.workerStarter = new WorkerStarter(this);
        n = this.workerStarter.startWorker();
    } catch (InitNodeException e) {
        ErrorManager.warn("There was an exception when initiating worker " + getName() + ".", e);
        throw e;
    }
    this.node = n;
    this.started = true;
    if (NIOTracer.isActivated()) {
        LOGGER.debug("Initializing NIO tracer " + this.getName());
        NIOTracer.startTracing(this.getName(), this.getUser(), this.getHost(), this.getLimitOfTasks());
    }
}

Example 5 with NIONode

use of es.bsc.comm.nio.NIONode in project compss by bsc-wdc.

the class NIOAdaptor method init.

@Override
public void init() {
    LOGGER.info("Initializing NIO Adaptor...");
    this.masterNode = new NIONode(null, MASTER_PORT);
    // Instantiate the NIO Message Handler
    final NIOMessageHandler mhm = new NIOMessageHandler(this);
    // Init the Transfer Manager
    LOGGER.debug("  Initializing the TransferManager structures...");
    try {
        TM.init(NIO_EVENT_MANAGER_CLASS, null, mhm);
    } catch (CommException ce) {
        String errMsg = "Error initializing the TransferManager";
        ErrorManager.error(errMsg, ce);
    }
    /* Init tracing values */
    tracing = System.getProperty(COMPSsConstants.TRACING) != null && Integer.parseInt(System.getProperty(COMPSsConstants.TRACING)) > 0;
    tracing_level = Integer.parseInt(System.getProperty(COMPSsConstants.TRACING));
    // Start the server
    LOGGER.debug("  Starting transfer server...");
    try {
        TM.startServer(masterNode);
    } catch (CommException ce) {
        String errMsg = "Error starting transfer server";
        ErrorManager.error(errMsg, ce);
    }
    // Start the Transfer Manager thread (starts the EventManager)
    LOGGER.debug("  Starting TransferManager Thread");
    TM.start();
}