Examples with ParameterServerId com.tencent.angel.ps.ParameterServerId used on opensource projects

Example 51 with ParameterServerId

use of com.tencent.angel.ps.ParameterServerId in project angel by Tencent.

the class AMMatrixMetaManager method handlePartReport.

private void handlePartReport(ParameterServerId psId, int matrixId, PartReport partReport) {
    ParameterServerId master = matrixMetaManager.getMasterPs(matrixId, partReport.partId);
    if (!psId.equals(master)) {
        MatrixMeta matrixMeta = matrixMetaManager.getMatrixMeta(matrixId);
        if (matrixMeta == null) {
            return;
        }
        matrixMeta.getPartitionMeta(partReport.partId).addReplicationPS(psId);
        if (partReport.state == PartitionState.INITIALIZING) {
            addNeedRecoverPart(master, new RecoverPartKey(new PartitionKey(matrixId, partReport.partId), new PSLocation(psId, context.getLocationManager().getPsLocation(psId))));
        } else if (partReport.state == PartitionState.READ_AND_WRITE) {
            ParameterServerId orignalMaster = matrixPartitionsOnPS.get(psId).get(matrixId).getPartitionMeta(partReport.partId).getMasterPs();
            if (orignalMaster.equals(psId)) {
                matrixMetaManager.getMatrixMeta(matrixId).getPartitionMeta(partReport.partId).makePsToMaster(psId);
            }
        }
    }
}

Example 52 with ParameterServerId

use of com.tencent.angel.ps.ParameterServerId in project angel by Tencent.

the class AMMatrixMetaManager method isCreated.

public boolean isCreated(int matrixId) {
    boolean inited = true;
    try {
        readLock.lock();
        if (!matrixMetaManager.exists(matrixId)) {
            return false;
        }
        Set<ParameterServerId> psIdSet = matrixIdToPSSetMap.get(matrixId);
        if (psIdSet == null || psIdSet.isEmpty()) {
            return false;
        }
        inited = true;
        for (ParameterServerId psId : psIdSet) {
            if (!psIdToMatrixIdsMap.containsKey(psId) || !psIdToMatrixIdsMap.get(psId).contains(matrixId)) {
                inited = false;
                break;
            }
        }
    } finally {
        readLock.unlock();
    }
    return inited;
}

Example 53 with ParameterServerId

use of com.tencent.angel.ps.ParameterServerId in project angel by Tencent.

the class AMModelLoader method split.

private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
    Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
    Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
    FileSystem fs = metaFilePath.getFileSystem(context.getConf());
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read matrix meta failed ", e);
        } finally {
            input.close();
        }
    } else {
        throw new IOException("Can not find meta file " + metaFilePath);
    }
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
        ParameterServerId psId = partEntry.getValue().getMasterPs();
        if (psId == null) {
            throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
        }
        Set partIds = psIdToPartIdsMap.get(psId);
        if (partIds == null) {
            partIds = new HashSet();
            psIdToPartIdsMap.put(psId, partIds);
        }
        partIds.add(partEntry.getKey());
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
        ret.put(entry.getKey(), psMatrixLoadContext);
    }
    return ret;
}

Example 54 with ParameterServerId

use of com.tencent.angel.ps.ParameterServerId in project angel by Tencent.

the class AppStateStorage method loadPSMeta.

/**
 * load ps meta from file
 *
 * @return Map<ParameterServerId,Integer> psId to attempt index map
 * @throws IOException
 */
public Map<ParameterServerId, Integer> loadPSMeta() throws IOException {
    try {
        psMetaLock.lock();
        // find ps meta file
        Path psMetaFilePath = null;
        try {
            psMetaFilePath = findFilePathUsePrefix(psMetaFileNamePrefix);
        } catch (Exception x) {
            LOG.error("find ps meta file failed.", x);
            return null;
        }
        // if ps meta file does not exist, just return null
        if (psMetaFilePath == null) {
            return null;
        }
        // read ps meta from file and deserialize it
        FSDataInputStream inputStream = fs.open(psMetaFilePath);
        int size = inputStream.readInt();
        Map<ParameterServerId, Integer> idToNextAttemptIndexMap = new HashMap<ParameterServerId, Integer>(size);
        for (int i = 0; i < size; i++) {
            idToNextAttemptIndexMap.put(new ParameterServerId(inputStream.readInt()), inputStream.readInt());
        }
        inputStream.close();
        return idToNextAttemptIndexMap;
    } finally {
        psMetaLock.unlock();
    }
}

Example 55 with ParameterServerId

use of com.tencent.angel.ps.ParameterServerId in project angel by Tencent.

the class ParameterServerManager method startAllPS.

/**
 * Start all PS
 */
public void startAllPS() {
    if (useMiniBatch) {
        Thread requestThread = new RequestThread();
        requestThread.setName("resource-requester");
        requestThread.start();
    } else {
        for (Map.Entry<ParameterServerId, AMParameterServer> entry : psMap.entrySet()) {
            entry.getValue().handle(new AMParameterServerEvent(AMParameterServerEventType.PS_SCHEDULE, entry.getKey()));
        }
    }
}