Search in sources :

Example 1 with PSMatrixSaveContext

use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.

the class AMModelSaver method split.

private Map<ParameterServerId, PSMatrixSaveContext> split(MatrixSaveContext matrixSaveContext) {
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixSaveContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixSaveContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    List<Integer> rowIndexes = matrixSaveContext.getRowIndexes();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    if (rowIndexes == null || rowIndexes.isEmpty()) {
        for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
            ParameterServerId psId = partEntry.getValue().getMasterPs();
            if (psId == null) {
                throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
            }
            Set partIds = psIdToPartIdsMap.get(psId);
            if (partIds == null) {
                partIds = new HashSet();
                psIdToPartIdsMap.put(psId, partIds);
            }
            partIds.add(partEntry.getKey());
        }
    } else {
        int size = rowIndexes.size();
        for (int i = 0; i < size; i++) {
            for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
                if (!partEntry.getValue().contain(rowIndexes.get(i))) {
                    continue;
                }
                ParameterServerId psId = partEntry.getValue().getMasterPs();
                if (psId == null) {
                    throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
                }
                Set partIds = psIdToPartIdsMap.get(psId);
                if (partIds == null) {
                    partIds = new HashSet();
                    psIdToPartIdsMap.put(psId, partIds);
                }
                partIds.add(partEntry.getKey());
            }
        }
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixSaveContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixSaveContext psMatrixSaveContext = new PSMatrixSaveContext(matrixId, partIds, matrixSaveContext.getRowIndexes(), matrixSaveContext.getFormatClassName(), null, false, true);
        ret.put(entry.getKey(), psMatrixSaveContext);
    }
    return ret;
}
Also used : HashSet(java.util.HashSet) Set(java.util.Set) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) PSMatrixSaveContext(com.tencent.angel.model.PSMatrixSaveContext) ArrayList(java.util.ArrayList) PartitionMeta(com.tencent.angel.ml.matrix.PartitionMeta) AMMatrixMetaManager(com.tencent.angel.master.matrixmeta.AMMatrixMetaManager) ParameterServerId(com.tencent.angel.ps.ParameterServerId) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashSet(java.util.HashSet)

Example 2 with PSMatrixSaveContext

use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.

the class SnapshotDumper method checkpoint.

private void checkpoint(List<Integer> matrixIds, int checkpointId, boolean cloneFirst, boolean sortFirst) throws IOException {
    if (matrixIds == null || matrixIds.isEmpty()) {
        return;
    }
    List<PSMatrixSaveContext> saveContexts = new ArrayList<>(matrixIds.size());
    List<Path> checkpointItemPaths = new ArrayList<>(matrixIds.size());
    List<Path> tempPaths = new ArrayList<>(matrixIds.size());
    for (int matrixId : matrixIds) {
        Path checkpointItemPath = genCheckpointPath(matrixId, checkpointId);
        Path tempPath = genTmpCheckpointPath(checkpointItemPath);
        checkpointItemPaths.add(checkpointItemPath);
        tempPaths.add(tempPath);
        MatrixMeta meta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
        saveContexts.add(new PSMatrixSaveContext(matrixId, new ArrayList<>(meta.getPartitionMetas().keySet()), null, SnapshotFormat.class.getName(), tempPath.toString(), cloneFirst, sortFirst));
    }
    context.getIOExecutors().save(new PSMatricesSaveContext(-1, -1, saveContexts), dumpParallel);
    // Rename temp to item path
    FileSystem fs = baseDirPath.getFileSystem(context.getConf());
    for (int i = 0; i < matrixIds.size(); i++) {
        HdfsUtil.rename(tempPaths.get(i), checkpointItemPaths.get(i), fs);
        clearOldCheckpoint(fs, genMatrixPath(matrixIds.get(0)));
    }
}
Also used : Path(org.apache.hadoop.fs.Path) PSMatricesSaveContext(com.tencent.angel.model.PSMatricesSaveContext) MatrixMeta(com.tencent.angel.ml.matrix.MatrixMeta) FileSystem(org.apache.hadoop.fs.FileSystem) PSMatrixSaveContext(com.tencent.angel.model.PSMatrixSaveContext) ArrayList(java.util.ArrayList)

Example 3 with PSMatrixSaveContext

use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.

the class PSModelIOExecutor method save.

/**
 * Save matrices to files
 *
 * @param saveContext matrices save context
 */
public void save(PSMatricesSaveContext saveContext, int parallel) throws IOException {
    LOG.info("start to save matrices :" + saveContext);
    Vector<String> errorLogs = new Vector<>();
    // Create and start workers
    IOExecutors workers = createIOExecutors(parallel);
    workers.init();
    workers.start();
    // Set workers
    for (PSMatrixSaveContext matrixSaveContext : saveContext.getMatrixSaveContexts()) {
        matrixSaveContext.setWorkers(workers);
    }
    try {
        MatrixDiskIOOp commitOp = new MatrixDiskIOOp(ACTION.SAVE, errorLogs, saveContext, 0, saveContext.getMatrixSaveContexts().size());
        workers.execute(commitOp);
        commitOp.join();
        if (!errorLogs.isEmpty()) {
            throw new IOException(StringUtils.join("\n", errorLogs));
        }
    } catch (Throwable x) {
        throw new IOException(x);
    } finally {
        workers.shutdown();
    }
    return;
}
Also used : IOExecutors(com.tencent.angel.model.io.IOExecutors) PSMatrixSaveContext(com.tencent.angel.model.PSMatrixSaveContext) IOException(java.io.IOException) Vector(java.util.Vector)

Example 4 with PSMatrixSaveContext

use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.

the class AMModelSaver method split.

private Map<ParameterServerId, PSMatricesSaveContext> split(int requestId, ModelSaveContext saveContext) {
    List<MatrixSaveContext> matricesContext = saveContext.getMatricesContext();
    Map<ParameterServerId, List<PSMatrixSaveContext>> psIdToContextsMap = new HashMap<>();
    int size = matricesContext.size();
    for (int i = 0; i < size; i++) {
        Map<ParameterServerId, PSMatrixSaveContext> psIdToContextMap = split(matricesContext.get(i));
        for (Map.Entry<ParameterServerId, PSMatrixSaveContext> matrixEntry : psIdToContextMap.entrySet()) {
            List<PSMatrixSaveContext> contexts = psIdToContextsMap.get(matrixEntry.getKey());
            if (contexts == null) {
                contexts = new ArrayList<>();
                psIdToContextsMap.put(matrixEntry.getKey(), contexts);
            }
            contexts.add(matrixEntry.getValue());
        }
    }
    Map<ParameterServerId, PSMatricesSaveContext> ret = new HashMap<>(psIdToContextsMap.size());
    int subRequestId = 0;
    for (Map.Entry<ParameterServerId, List<PSMatrixSaveContext>> modelEntry : psIdToContextsMap.entrySet()) {
        Path psPath = new Path(new Path(new Path(saveContext.getTmpSavePath()), ModelFilesConstent.resultDirName), modelEntry.getKey().toString());
        List<PSMatrixSaveContext> psMatrixContexts = modelEntry.getValue();
        for (PSMatrixSaveContext matrixContext : psMatrixContexts) {
            matrixContext.setSavePath(new Path(psPath, context.getMatrixMetaManager().getMatrix(matrixContext.getMatrixId()).getName()).toString());
        }
        ret.put(modelEntry.getKey(), new PSMatricesSaveContext(requestId, subRequestId++, modelEntry.getValue()));
    }
    return ret;
}
Also used : Path(org.apache.hadoop.fs.Path) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) PSMatrixSaveContext(com.tencent.angel.model.PSMatrixSaveContext) PSMatrixSaveContext(com.tencent.angel.model.PSMatrixSaveContext) MatrixSaveContext(com.tencent.angel.model.MatrixSaveContext) PSMatricesSaveContext(com.tencent.angel.model.PSMatricesSaveContext) ArrayList(java.util.ArrayList) List(java.util.List) ParameterServerId(com.tencent.angel.ps.ParameterServerId) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

PSMatrixSaveContext (com.tencent.angel.model.PSMatrixSaveContext)4 ArrayList (java.util.ArrayList)3 MatrixMeta (com.tencent.angel.ml.matrix.MatrixMeta)2 PSMatricesSaveContext (com.tencent.angel.model.PSMatricesSaveContext)2 ParameterServerId (com.tencent.angel.ps.ParameterServerId)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 Path (org.apache.hadoop.fs.Path)2 AMMatrixMetaManager (com.tencent.angel.master.matrixmeta.AMMatrixMetaManager)1 PartitionMeta (com.tencent.angel.ml.matrix.PartitionMeta)1 MatrixSaveContext (com.tencent.angel.model.MatrixSaveContext)1 IOExecutors (com.tencent.angel.model.io.IOExecutors)1 IOException (java.io.IOException)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Set (java.util.Set)1 Vector (java.util.Vector)1 FileSystem (org.apache.hadoop.fs.FileSystem)1