Examples with InvalidParameterException com.tencent.angel.exception.InvalidParameterException used on opensource projects

Example 16 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class IndexGetRowTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient matrixClient = taskContext.getMatrix(IndexGetRowTest.DENSE_DOUBLE_MAT);
        MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(IndexGetRowTest.DENSE_DOUBLE_MAT);
        int[] indices = genIndexs((int) matrixMeta.getColNum(), IndexGetRowTest.nnz);
        IntDoubleVector delta = VFactory.sparseDoubleVector((int) matrixMeta.getColNum(), IndexGetRowTest.nnz);
        for (int i = 0; i < IndexGetRowTest.nnz; i++) {
            delta.set(indices[i], indices[i]);
        }
        IntDoubleVector delta1 = VFactory.denseDoubleVector((int) matrixMeta.getColNum());
        for (int i = 0; i < IndexGetRowTest.colNum; i++) {
            delta1.set(i, i);
        }
        LOG.info("delta use " + delta.getType() + " type storage");
        int testNum = 500;
        long startTs = System.currentTimeMillis();
        LOG.info("for sparse delta type");
        conf.setBoolean("use.new.split", false);
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta, true);
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment old use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        conf.setBoolean("use.new.split", true);
        startTs = System.currentTimeMillis();
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta, true);
            // IntDoubleVector vector = (IntDoubleVector) ((GetRowResult) matrixClient.get(func)).getRow();
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment new use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        LOG.info("for dense delta type");
        conf.setBoolean("use.new.split", false);
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta1, true);
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment old use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
        conf.setBoolean("use.new.split", true);
        startTs = System.currentTimeMillis();
        for (int i = 0; i < testNum; i++) {
            matrixClient.increment(0, delta1, true);
            // IntDoubleVector vector = (IntDoubleVector) ((GetRowResult) matrixClient.get(func)).getRow();
            if (i > 0 && i % 10 == 0) {
                LOG.info("increment new use time = " + (System.currentTimeMillis() - startTs) / i);
            }
        }
    } catch (InvalidParameterException e) {
        LOG.error("get matrix failed ", e);
        throw new AngelException(e);
    }
}

Example 17 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class ConfUtils method initConf.

public static Configuration initConf(String[] cmdArgs) throws Exception {
    // Parse cmd parameters
    Map<String, String> cmdConfMap = parseArgs(cmdArgs);
    // load hadoop configuration
    final Configuration conf = new Configuration();
    String hadoopHomePath = System.getenv("HADOOP_HOME");
    if (hadoopHomePath == null) {
        LOG.warn("HADOOP_HOME is empty.");
    } else {
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/yarn-site.xml"));
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/hdfs-site.xml"));
    }
    // load angel system configuration
    String angelHomePath = System.getenv("ANGEL_HOME");
    if (angelHomePath == null) {
        LOG.fatal("ANGEL_HOME is empty, please set it first");
        throw new InvalidParameterException("ANGEL_HOME is empty, please set it first");
    }
    LOG.info("angelHomePath conf path=" + angelHomePath + "/conf/" + angelSysConfFile);
    conf.addResource(new Path(angelHomePath + "/conf/" + angelSysConfFile));
    LOG.info("load system config file success");
    // load user configuration:
    // load user config file
    String jobConfFile = cmdConfMap.get(AngelConf.ANGEL_APP_CONFIG_FILE);
    if (jobConfFile != null) {
        LOG.info("user app config file " + jobConfFile);
        conf.addResource(new Path(jobConfFile));
    } else {
        jobConfFile = conf.get(AngelConf.ANGEL_APP_CONFIG_FILE);
        if (jobConfFile != null) {
            LOG.info("user app config file " + jobConfFile);
            conf.addResource(new Path(jobConfFile));
        }
    }
    // load command line parameters
    if (cmdConfMap != null && !cmdConfMap.isEmpty()) {
        for (Map.Entry<String, String> entry : cmdConfMap.entrySet()) {
            conf.set(entry.getKey(), entry.getValue());
        }
    }
    // load user job resource files
    String userResourceFiles = conf.get(AngelConf.ANGEL_APP_USER_RESOURCE_FILES);
    if (userResourceFiles != null) {
        addResourceFiles(conf, userResourceFiles);
    }
    // load ml conf file for graph based algorithm
    String mlConfFiles = conf.get(AngelConf.ANGEL_ML_CONF);
    if (mlConfFiles != null && mlConfFiles.length() != 0) {
        addResourceFiles(conf, mlConfFiles);
    }
    // load user job jar if it exist
    String jobJar = conf.get(AngelConf.ANGEL_JOB_JAR);
    if (jobJar != null) {
        loadJar(jobJar);
        addResourceFiles(conf, jobJar);
    }
    // Expand the environment variable
    try {
        expandEnv(conf);
    } catch (Exception x) {
        LOG.warn("expand env in configuration failed.", x);
    }
    return conf;
}

Example 18 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class NetUtils method chooseAListenPort.

public static int chooseAListenPort(Configuration conf) throws IOException {
    String portRangeStr = conf.get(AngelConf.ANGEL_LISTEN_PORT_RANGE, AngelConf.DEFAULT_ANGEL_LISTEN_PORT_RANGE);
    String[] portRangeArray = null;
    int startPort = -1;
    int endPort = -1;
    try {
        portRangeArray = portRangeStr.split(",");
        startPort = Integer.valueOf(portRangeArray[0]);
        endPort = Integer.valueOf(portRangeArray[1]);
        if (startPort <= 1024 || startPort > 65535 || endPort <= 1024 || endPort > 65535 || startPort > endPort) {
            throw new InvalidParameterException(AngelConf.ANGEL_LISTEN_PORT_RANGE, portRangeStr, "port should in range 1024~63335");
        }
    } catch (Exception x) {
        LOG.error("use port set for " + AngelConf.ANGEL_LISTEN_PORT_RANGE + " is unvalid, we use default value now. error msg = " + x.getMessage());
        portRangeArray = AngelConf.DEFAULT_ANGEL_LISTEN_PORT_RANGE.split(",");
        startPort = Integer.valueOf(portRangeArray[0]);
        endPort = Integer.valueOf(portRangeArray[1]);
    }
    int maxTryTime = 100;
    Random r = new Random();
    int port = -1;
    for (int i = 0; i < maxTryTime; i++) {
        port = Math.abs(r.nextInt()) % (endPort - startPort) + startPort;
        if (isPortAvailable(port)) {
            return port;
        }
        LOG.error("workerservice:port " + port + " is not available, try agine");
    }
    throw new IOException("can not find a avaliable port for workerservice");
}

Example 19 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class MatrixRowGetTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient matrixClient = taskContext.getMatrix(MatrixRowGetTest.SPARSE_LONGKEY_FLOAT_MAT);
        MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(MatrixRowGetTest.SPARSE_LONGKEY_FLOAT_MAT);
        long[] indices = genLongIndexs(matrixMeta.getColNum(), MatrixRowGetTest.longnnz);
        long[] getIndices = genGetLongIndexs(indices, MatrixRowGetTest.nGet);
        LongFloatVector deltatest = VFactory.sparseLongKeyFloatVector(matrixMeta.getColNum(), MatrixRowGetTest.longnnz);
        for (int i = 0; i < MatrixRowGetTest.longnnz; i++) {
            deltatest.set(indices[i], (float) (indices[i] * 1.3));
        }
        deltatest.setRowId(0);
        LOG.info(" delta use " + deltatest.getType() + " type storage");
        int updateTime = 0;
        long startTs = System.currentTimeMillis();
        try {
            matrixClient.increment(0, deltatest, true);
            long startoneTs = System.currentTimeMillis();
            LOG.info("increment time= " + (startoneTs - startTs));
            while (true) {
                LongFloatVector temp = (LongFloatVector) matrixClient.get(0, getIndices);
                updateTime++;
                if (updateTime % 10 == 0) {
                    LOG.info("get times = " + updateTime + ", avg update time=" + (System.currentTimeMillis() - startoneTs) / updateTime);
                }
            }
        } catch (Throwable ie) {
            LOG.info("mistake happened in MatrixRowGet try");
        }
    } catch (InvalidParameterException e) {
        LOG.error("get matrix failed ", e);
        throw new AngelException(e);
    }
}

Example 20 with InvalidParameterException

use of com.tencent.angel.exception.InvalidParameterException in project angel by Tencent.

the class MatrixRowGetUDFTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient matrixClient = taskContext.getMatrix(MatrixRowGetTest.SPARSE_LONGKEY_FLOAT_MAT);
        MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(MatrixRowGetTest.SPARSE_LONGKEY_FLOAT_MAT);
        long[] indices = genLongIndexs(matrixMeta.getColNum(), MatrixRowGetTest.longnnz);
        long[] getIndices = genGetLongIndexs(indices, MatrixRowGetTest.nGet);
        LongFloatVector deltatest = VFactory.sparseLongKeyFloatVector(matrixMeta.getColNum(), MatrixRowGetTest.longnnz);
        for (int i = 0; i < MatrixRowGetTest.longnnz; i++) {
            deltatest.set(indices[i], (float) (indices[i] * 1.3));
        }
        deltatest.setRowId(0);
        LOG.info(" delta use " + deltatest.getType() + " type storage");
        int updateTime = 0;
        long startTs = System.currentTimeMillis();
        try {
            matrixClient.increment(0, deltatest, true);
            long startoneTs = System.currentTimeMillis();
            LOG.info("increment time= " + (startoneTs - startTs));
            while (true) {
                LongIndexGet funca = new LongIndexGet(new LongIndexGetParam(matrixClient.getMatrixId(), 0, getIndices));
                LongFloatVector tempOne = (LongFloatVector) ((GetRowResult) PSAgentContext.get().getUserRequestAdapter().get(funca).get()).getRow();
                updateTime++;
                if (updateTime % 10 == 0) {
                    LOG.info("get times = " + updateTime + ", avg update time=" + (System.currentTimeMillis() - startoneTs) / updateTime);
                }
            }
        } catch (Throwable ie) {
            LOG.info("mistake happened in MatrixRowGet try");
        }
    } catch (InvalidParameterException e) {
        LOG.error("get matrix failed ", e);
        throw new AngelException(e);
    }
}