Search in sources :

Example 21 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class GetLabels method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    GeneralPartGetParam param = (GeneralPartGetParam) partParam;
    KeyPart keyPart = param.getIndicesPart();
    switch(keyPart.getKeyType()) {
        case LONG:
            {
                // Long type node id
                long[] nodeIds = ((ILongKeyPartOp) keyPart).getKeys();
                ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
                Long2ObjectOpenHashMap<float[]> nodeIdToLabels = new Long2ObjectOpenHashMap<>(nodeIds.length);
                for (long nodeId : nodeIds) {
                    if (row.get(nodeId) == null) {
                        // If node not exist, just skip
                        continue;
                    }
                    float[] labels = ((GraphNode) (row.get(nodeId))).getLabels();
                    if (labels != null) {
                        nodeIdToLabels.put(nodeId, labels);
                    }
                }
                return new PartGetFloatArrayAttrsResult(param.getPartKey().getPartitionId(), nodeIdToLabels);
            }
        default:
            {
                // TODO: support String, Int, and Any type node id
                throw new InvalidParameterException("Unsupport index type " + keyPart.getKeyType());
            }
    }
}
Also used : InvalidParameterException(com.tencent.angel.exception.InvalidParameterException) KeyPart(com.tencent.angel.psagent.matrix.transport.router.KeyPart) Long2ObjectOpenHashMap(it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap) PartGetFloatArrayAttrsResult(com.tencent.angel.graph.client.psf.get.utils.PartGetFloatArrayAttrsResult) ServerLongAnyRow(com.tencent.angel.ps.storage.vector.ServerLongAnyRow)

Example 22 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class GetNodeFeats method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    GeneralPartGetParam param = (GeneralPartGetParam) partParam;
    KeyPart keyPart = param.getIndicesPart();
    switch(keyPart.getKeyType()) {
        case LONG:
            {
                // Long type node id
                long[] nodeIds = ((ILongKeyPartOp) keyPart).getKeys();
                ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
                Long2ObjectOpenHashMap<IntFloatVector> nodeIdToFeats = new Long2ObjectOpenHashMap<>(nodeIds.length);
                for (long nodeId : nodeIds) {
                    if (row.get(nodeId) == null) {
                        // If node not exist, just skip
                        continue;
                    }
                    IntFloatVector feat = ((GraphNode) (row.get(nodeId))).getFeats();
                    if (feat != null) {
                        nodeIdToFeats.put(nodeId, feat);
                    }
                }
                return new PartGetNodeFeatsResult(param.getPartKey().getPartitionId(), nodeIdToFeats);
            }
        default:
            {
                // TODO: support String, Int, and Any type node id
                throw new InvalidParameterException("Unsupport index type " + keyPart.getKeyType());
            }
    }
}
Also used : InvalidParameterException(com.tencent.angel.exception.InvalidParameterException) GeneralPartGetParam(com.tencent.angel.ml.matrix.psf.get.base.GeneralPartGetParam) KeyPart(com.tencent.angel.psagent.matrix.transport.router.KeyPart) Long2ObjectOpenHashMap(it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap) ServerLongAnyRow(com.tencent.angel.ps.storage.vector.ServerLongAnyRow) IntFloatVector(com.tencent.angel.ml.math2.vector.IntFloatVector)

Example 23 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class ModelConverterTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        // Get input path, output path
        String modelLoadDir = conf.get(AngelConf.ANGEL_LOAD_MODEL_PATH);
        if (modelLoadDir == null) {
            throw new InvalidParameterException("convert source path " + AngelConf.ANGEL_LOAD_MODEL_PATH + " must be set");
        }
        String convertedModelSaveDir = conf.get(AngelConf.ANGEL_SAVE_MODEL_PATH);
        if (convertedModelSaveDir == null) {
            throw new InvalidParameterException("converted model save path " + AngelConf.ANGEL_LOAD_MODEL_PATH + " must be set");
        }
        // Init serde
        String modelSerdeClass = conf.get("angel.modelconverts.serde.class", TextModelLineConvert.class.getName());
        Class<? extends ModelLineConvert> funcClass = (Class<? extends ModelLineConvert>) Class.forName(modelSerdeClass);
        Constructor<? extends ModelLineConvert> constructor = funcClass.getConstructor();
        constructor.setAccessible(true);
        ModelLineConvert serde = constructor.newInstance();
        // Parse need convert model names, if not set, we will convert all models in input directory
        String needConvertModelNames = conf.get("angel.modelconverts.model.names");
        String[] modelNames = null;
        if (needConvertModelNames == null) {
            LOG.info("we will convert all models save in " + modelLoadDir);
            Path modelLoadPath = new Path(modelLoadDir);
            FileSystem fs = modelLoadPath.getFileSystem(conf);
            FileStatus[] fileStatus = fs.listStatus(modelLoadPath);
            if (fileStatus == null || fileStatus.length == 0) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
            List<String> modelNameList = new ArrayList<>();
            for (int i = 0; i < fileStatus.length; i++) {
                if (fileStatus[i].isDirectory()) {
                    modelNameList.add(fileStatus[i].getPath().getName());
                }
            }
            if (modelNameList.isEmpty()) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
            modelNames = modelNameList.toArray(new String[0]);
        } else {
            modelNames = needConvertModelNames.split(",");
            if (modelNames.length == 0) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
        }
        for (int i = 0; i < modelNames.length; i++) {
            LOG.info("===================start to convert model " + modelNames[i]);
            ModelConverter.convert(conf, modelLoadDir + Path.SEPARATOR + modelNames[i], convertedModelSaveDir + Path.SEPARATOR + modelNames[i], serde);
            LOG.info("===================end to convert model " + modelNames[i]);
        }
    } catch (Throwable x) {
        LOG.fatal("convert model falied, ", x);
        throw new AngelException(x);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) AngelException(com.tencent.angel.exception.AngelException) FileStatus(org.apache.hadoop.fs.FileStatus) ArrayList(java.util.ArrayList) TextModelLineConvert(com.tencent.angel.tools.TextModelLineConvert) TextModelLineConvert(com.tencent.angel.tools.TextModelLineConvert) ModelLineConvert(com.tencent.angel.tools.ModelLineConvert) IOException(java.io.IOException) InvalidParameterException(com.tencent.angel.exception.InvalidParameterException) FileSystem(org.apache.hadoop.fs.FileSystem)

Example 24 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class PSAgentManager method init.

public void init() throws InvalidParameterException {
    Configuration conf = context.getConf();
    if (context.getRunningMode() == RunningMode.ANGEL_PS_PSAGENT) {
        String ipStr = conf.get(AngelConf.ANGEL_PSAGENT_IPLIST);
        if (ipStr != null) {
            ips = ipStr.split(",");
            psAgentNumber = ips.length;
        } else {
            throw new InvalidParameterException("ip list is null, property " + AngelConf.ANGEL_PSAGENT_IPLIST + " must be set");
        }
        initPSAgents();
    } else {
        ips = null;
        psAgentNumber = 0;
    }
}
Also used : InvalidParameterException(com.tencent.angel.exception.InvalidParameterException) Configuration(org.apache.hadoop.conf.Configuration)

Aggregations

InvalidParameterException (com.tencent.angel.exception.InvalidParameterException)24 KeyPart (com.tencent.angel.psagent.matrix.transport.router.KeyPart)8 ServerLongAnyRow (com.tencent.angel.ps.storage.vector.ServerLongAnyRow)7 Long2ObjectOpenHashMap (it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap)7 AngelException (com.tencent.angel.exception.AngelException)6 IOException (java.io.IOException)6 Path (org.apache.hadoop.fs.Path)4 MatrixMeta (com.tencent.angel.ml.matrix.MatrixMeta)3 GeneralPartGetParam (com.tencent.angel.ml.matrix.psf.get.base.GeneralPartGetParam)3 MatrixClient (com.tencent.angel.psagent.matrix.MatrixClient)3 PartGetFloatArrayAttrsResult (com.tencent.angel.graph.client.psf.get.utils.PartGetFloatArrayAttrsResult)2 PartGetIntArrayAttrsResult (com.tencent.angel.graph.client.psf.get.utils.PartGetIntArrayAttrsResult)2 IntFloatVector (com.tencent.angel.ml.math2.vector.IntFloatVector)2 LongFloatVector (com.tencent.angel.ml.math2.vector.LongFloatVector)2 MatrixContext (com.tencent.angel.ml.matrix.MatrixContext)2 ModelLineConvert (com.tencent.angel.tools.ModelLineConvert)2 TextModelLineConvert (com.tencent.angel.tools.TextModelLineConvert)2 FileNotFoundException (java.io.FileNotFoundException)2 ArrayList (java.util.ArrayList)2 Configuration (org.apache.hadoop.conf.Configuration)2