use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.
the class AMModelSaver method split.
private Map<ParameterServerId, PSMatrixSaveContext> split(MatrixSaveContext matrixSaveContext) {
AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
MatrixMeta meta = matrixMetaManager.getMatrix(matrixSaveContext.getMatrixName());
if (meta == null) {
throw new IllegalStateException("Can not find matrix " + matrixSaveContext.getMatrixName());
}
Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
List<Integer> rowIndexes = matrixSaveContext.getRowIndexes();
Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
if (rowIndexes == null || rowIndexes.isEmpty()) {
for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
ParameterServerId psId = partEntry.getValue().getMasterPs();
if (psId == null) {
throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
}
Set partIds = psIdToPartIdsMap.get(psId);
if (partIds == null) {
partIds = new HashSet();
psIdToPartIdsMap.put(psId, partIds);
}
partIds.add(partEntry.getKey());
}
} else {
int size = rowIndexes.size();
for (int i = 0; i < size; i++) {
for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
if (!partEntry.getValue().contain(rowIndexes.get(i))) {
continue;
}
ParameterServerId psId = partEntry.getValue().getMasterPs();
if (psId == null) {
throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
}
Set partIds = psIdToPartIdsMap.get(psId);
if (partIds == null) {
partIds = new HashSet();
psIdToPartIdsMap.put(psId, partIds);
}
partIds.add(partEntry.getKey());
}
}
}
int matrixId = meta.getId();
Map<ParameterServerId, PSMatrixSaveContext> ret = new HashMap<>(psIdToPartIdsMap.size());
for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
List<Integer> partIds = new ArrayList<>(entry.getValue());
partIds.sort(new Comparator<Integer>() {
@Override
public int compare(Integer id1, Integer id2) {
return id1 - id2;
}
});
PSMatrixSaveContext psMatrixSaveContext = new PSMatrixSaveContext(matrixId, partIds, matrixSaveContext.getRowIndexes(), matrixSaveContext.getFormatClassName(), null, false, true);
ret.put(entry.getKey(), psMatrixSaveContext);
}
return ret;
}
use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.
the class SnapshotDumper method checkpoint.
private void checkpoint(List<Integer> matrixIds, int checkpointId, boolean cloneFirst, boolean sortFirst) throws IOException {
if (matrixIds == null || matrixIds.isEmpty()) {
return;
}
List<PSMatrixSaveContext> saveContexts = new ArrayList<>(matrixIds.size());
List<Path> checkpointItemPaths = new ArrayList<>(matrixIds.size());
List<Path> tempPaths = new ArrayList<>(matrixIds.size());
for (int matrixId : matrixIds) {
Path checkpointItemPath = genCheckpointPath(matrixId, checkpointId);
Path tempPath = genTmpCheckpointPath(checkpointItemPath);
checkpointItemPaths.add(checkpointItemPath);
tempPaths.add(tempPath);
MatrixMeta meta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
saveContexts.add(new PSMatrixSaveContext(matrixId, new ArrayList<>(meta.getPartitionMetas().keySet()), null, SnapshotFormat.class.getName(), tempPath.toString(), cloneFirst, sortFirst));
}
context.getIOExecutors().save(new PSMatricesSaveContext(-1, -1, saveContexts), dumpParallel);
// Rename temp to item path
FileSystem fs = baseDirPath.getFileSystem(context.getConf());
for (int i = 0; i < matrixIds.size(); i++) {
HdfsUtil.rename(tempPaths.get(i), checkpointItemPaths.get(i), fs);
clearOldCheckpoint(fs, genMatrixPath(matrixIds.get(0)));
}
}
use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.
the class PSModelIOExecutor method save.
/**
* Save matrices to files
*
* @param saveContext matrices save context
*/
public void save(PSMatricesSaveContext saveContext, int parallel) throws IOException {
LOG.info("start to save matrices :" + saveContext);
Vector<String> errorLogs = new Vector<>();
// Create and start workers
IOExecutors workers = createIOExecutors(parallel);
workers.init();
workers.start();
// Set workers
for (PSMatrixSaveContext matrixSaveContext : saveContext.getMatrixSaveContexts()) {
matrixSaveContext.setWorkers(workers);
}
try {
MatrixDiskIOOp commitOp = new MatrixDiskIOOp(ACTION.SAVE, errorLogs, saveContext, 0, saveContext.getMatrixSaveContexts().size());
workers.execute(commitOp);
commitOp.join();
if (!errorLogs.isEmpty()) {
throw new IOException(StringUtils.join("\n", errorLogs));
}
} catch (Throwable x) {
throw new IOException(x);
} finally {
workers.shutdown();
}
return;
}
use of com.tencent.angel.model.PSMatrixSaveContext in project angel by Tencent.
the class AMModelSaver method split.
private Map<ParameterServerId, PSMatricesSaveContext> split(int requestId, ModelSaveContext saveContext) {
List<MatrixSaveContext> matricesContext = saveContext.getMatricesContext();
Map<ParameterServerId, List<PSMatrixSaveContext>> psIdToContextsMap = new HashMap<>();
int size = matricesContext.size();
for (int i = 0; i < size; i++) {
Map<ParameterServerId, PSMatrixSaveContext> psIdToContextMap = split(matricesContext.get(i));
for (Map.Entry<ParameterServerId, PSMatrixSaveContext> matrixEntry : psIdToContextMap.entrySet()) {
List<PSMatrixSaveContext> contexts = psIdToContextsMap.get(matrixEntry.getKey());
if (contexts == null) {
contexts = new ArrayList<>();
psIdToContextsMap.put(matrixEntry.getKey(), contexts);
}
contexts.add(matrixEntry.getValue());
}
}
Map<ParameterServerId, PSMatricesSaveContext> ret = new HashMap<>(psIdToContextsMap.size());
int subRequestId = 0;
for (Map.Entry<ParameterServerId, List<PSMatrixSaveContext>> modelEntry : psIdToContextsMap.entrySet()) {
Path psPath = new Path(new Path(new Path(saveContext.getTmpSavePath()), ModelFilesConstent.resultDirName), modelEntry.getKey().toString());
List<PSMatrixSaveContext> psMatrixContexts = modelEntry.getValue();
for (PSMatrixSaveContext matrixContext : psMatrixContexts) {
matrixContext.setSavePath(new Path(psPath, context.getMatrixMetaManager().getMatrix(matrixContext.getMatrixId()).getName()).toString());
}
ret.put(modelEntry.getKey(), new PSMatricesSaveContext(requestId, subRequestId++, modelEntry.getValue()));
}
return ret;
}
Aggregations