Examples with IOExecutors com.tencent.angel.model.io.IOExecutors used on opensource projects

Example 1 with IOExecutors

use of com.tencent.angel.model.io.IOExecutors in project angel by Tencent.

the class PSModelIOExecutor method load.

/**
 * Load matrices from files
 *
 * @param loadContext load context
 */
public void load(PSMatricesLoadContext loadContext, int parallel) throws IOException {
    LOG.info("start to load matrices :" + loadContext);
    Vector<String> errorLogs = new Vector<>();
    // Create and start workers
    IOExecutors workers = createIOExecutors(parallel);
    workers.init();
    workers.start();
    // Set workers
    for (PSMatrixLoadContext matrixLoadContext : loadContext.getMatrixLoadContexts()) {
        matrixLoadContext.setWorkers(workers);
    }
    try {
        MatrixDiskIOOp commitOp = new MatrixDiskIOOp(ACTION.LOAD, errorLogs, loadContext, 0, loadContext.getMatrixLoadContexts().size());
        workers.execute(commitOp);
        commitOp.join();
        if (!errorLogs.isEmpty()) {
            throw new IOException(StringUtils.join("\n", errorLogs));
        }
    } catch (Throwable x) {
        throw new IOException(x);
    } finally {
        workers.shutdown();
    }
    return;
}

Example 2 with IOExecutors

use of com.tencent.angel.model.io.IOExecutors in project angel by Tencent.

the class PSModelIOExecutor method save.

/**
 * Save matrices to files
 *
 * @param saveContext matrices save context
 */
public void save(PSMatricesSaveContext saveContext, int parallel) throws IOException {
    LOG.info("start to save matrices :" + saveContext);
    Vector<String> errorLogs = new Vector<>();
    // Create and start workers
    IOExecutors workers = createIOExecutors(parallel);
    workers.init();
    workers.start();
    // Set workers
    for (PSMatrixSaveContext matrixSaveContext : saveContext.getMatrixSaveContexts()) {
        matrixSaveContext.setWorkers(workers);
    }
    try {
        MatrixDiskIOOp commitOp = new MatrixDiskIOOp(ACTION.SAVE, errorLogs, saveContext, 0, saveContext.getMatrixSaveContexts().size());
        workers.execute(commitOp);
        commitOp.join();
        if (!errorLogs.isEmpty()) {
            throw new IOException(StringUtils.join("\n", errorLogs));
        }
    } catch (Throwable x) {
        throw new IOException(x);
    } finally {
        workers.shutdown();
    }
    return;
}

Example 3 with IOExecutors

use of com.tencent.angel.model.io.IOExecutors in project angel by Tencent.

the class AMModelSaver method combine.

private void combine(ModelSaveContext saveContext, ModelSaveResult result) throws IOException {
    Path tmpPath = new Path(saveContext.getTmpSavePath());
    final FileSystem fs = tmpPath.getFileSystem(context.getConf());
    final Path tmpCombinePath = HdfsUtil.toFinalPath(tmpPath);
    if (fs.exists(tmpCombinePath)) {
        fs.delete(tmpCombinePath, true);
    }
    combineDispatchThread = new Thread(new Runnable() {

        @Override
        public void run() {
            Vector<String> errorLogs = new Vector<>();
            ModelCombineOp op = new ModelCombineOp(saveContext, errorLogs, 0, saveContext.getMatricesContext().size(), tmpCombinePath, fs);
            fileOpExecutor = new IOExecutors(context.getConf().getInt(AngelConf.ANGEL_AM_MATRIX_DISKIO_WORKER_POOL_SIZE, AngelConf.DEFAULT_ANGEL_AM_MATRIX_DISKIO_WORKER_POOL_SIZE));
            fileOpExecutor.init();
            fileOpExecutor.start();
            try {
                fileOpExecutor.execute(op);
                op.join();
                if (!errorLogs.isEmpty()) {
                    String errorLog = "move output files for matrice failed, error msg = " + StringUtils.join(";", errorLogs);
                    LOG.error(errorLog);
                    saveFailed(result, errorLog);
                } else {
                    finalCommit(tmpCombinePath, new Path(saveContext.getSavePath()), fs);
                    saveSuccess(result);
                    recordSaveResult(saveContext);
                }
            } catch (Throwable x) {
                LOG.error("move output files for matrice failed. ", x);
                saveFailed(result, StringUtils.stringifyException(x));
            } finally {
                clean(saveContext);
            }
        }
    });
    combineDispatchThread.setName("CommitTaskDispacher");
    combineDispatchThread.start();
}

Example 4 with IOExecutors

use of com.tencent.angel.model.io.IOExecutors in project angel by Tencent.

the class AMMatrixCommitter method startCommitDispacherThread.

private void startCommitDispacherThread() {
    if (needSaveMatrixIds == null || needSaveMatrixIds.isEmpty()) {
        LOG.info("there are no matrices need save");
        return;
    }
    commitDispatchThread = new Thread(new Runnable() {

        @Override
        public void run() {
            Vector<String> errorLogs = new Vector<>();
            MatrixCommitOp op = new MatrixCommitOp(needSaveMatrixIds, errorLogs, 0, needSaveMatrixIds.size());
            fileOpExecutor = new IOExecutors(context.getConf().getInt(AngelConf.ANGEL_AM_MATRIX_DISKIO_WORKER_POOL_SIZE, AngelConf.DEFAULT_ANGEL_AM_MATRIX_DISKIO_WORKER_POOL_SIZE));
            fileOpExecutor.init();
            fileOpExecutor.start();
            try {
                fileOpExecutor.execute(op);
                op.join();
                if (!errorLogs.isEmpty()) {
                    String errorLog = "move output files for matrice failed, error msg = " + StringUtils.join(";", errorLogs);
                    LOG.error(errorLog);
                    commitFailed(errorLog);
                } else {
                    finalCommit();
                    commitSuccess();
                }
            } catch (Throwable x) {
                LOG.error("move output files for matrice failed. ", x);
                commitFailed("move output files for matrice failed, error msg = " + x.getMessage());
            }
        }
    });
    commitDispatchThread.setName("CommitTaskDispacher");
    commitDispatchThread.start();
}