use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.
the class MasterService method getMatrices.
/**
* Get matrices metadata
*/
@Override
public GetMatricesResponse getMatrices(RpcController controller, GetMatricesRequest request) throws ServiceException {
GetMatricesResponse.Builder builder = GetMatricesResponse.newBuilder();
AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
List<String> matrixNames = request.getMatrixNamesList();
int size = matrixNames.size();
for (int i = 0; i < size; i++) {
MatrixMeta matrixMeta = matrixMetaManager.getMatrix(matrixNames.get(i));
if (matrixMeta == null) {
throw new ServiceException("Can not find matrix " + matrixNames.get(i));
}
builder.addMatrixMetas(ProtobufUtil.convertToMatrixMetaProto(matrixMeta));
}
return builder.build();
}
use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.
the class AMModelSaver method getLoadPath.
private SaveResult getLoadPath(int matrixId) throws IOException {
SaveResult result = matrixIdToLoadPathResult.get(matrixId);
if (result == null) {
String loadPath = context.getConf().get(AngelConf.ANGEL_LOAD_MODEL_PATH);
if (loadPath != null) {
AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
MatrixMeta meta = matrixMetaManager.getMatrix(matrixId);
if (meta != null) {
Path matrixPath = new Path(loadPath, meta.getName());
FileSystem fs = matrixPath.getFileSystem(context.getConf());
if (fs.exists(matrixPath)) {
result = new SaveResult(loadPath, matrixPath.toString(), -1);
matrixIdToLoadPathResult.putIfAbsent(matrixId, result);
}
}
}
}
return result;
}
use of com.tencent.angel.master.matrixmeta.AMMatrixMetaManager in project angel by Tencent.
the class AMModelLoader method split.
private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
FileSystem fs = metaFilePath.getFileSystem(context.getConf());
if (fs.exists(metaFilePath)) {
FSDataInputStream input = fs.open(metaFilePath);
try {
matrixFilesMeta.read(input);
} catch (Throwable e) {
throw new IOException("Read matrix meta failed ", e);
} finally {
input.close();
}
} else {
throw new IOException("Can not find meta file " + metaFilePath);
}
AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
if (meta == null) {
throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
}
Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
ParameterServerId psId = partEntry.getValue().getMasterPs();
if (psId == null) {
throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
}
Set partIds = psIdToPartIdsMap.get(psId);
if (partIds == null) {
partIds = new HashSet();
psIdToPartIdsMap.put(psId, partIds);
}
partIds.add(partEntry.getKey());
}
int matrixId = meta.getId();
Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
List<Integer> partIds = new ArrayList<>(entry.getValue());
partIds.sort(new Comparator<Integer>() {
@Override
public int compare(Integer id1, Integer id2) {
return id1 - id2;
}
});
PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
ret.put(entry.getKey(), psMatrixLoadContext);
}
return ret;
}
Aggregations