Examples with AngelException com.tencent.angel.exception.AngelException used on opensource projects

Example 16 with AngelException

use of com.tencent.angel.exception.AngelException in project angel by Tencent.

the class LikelihoodFunc method likelihood.

private double likelihood(ServerRow row, float beta, double lgammaBeta) {
    int len = (int) (row.getEndCol() - row.getStartCol());
    double ll = 0;
    if (row instanceof ServerDenseIntRow) {
        IntBuffer buf = ((ServerDenseIntRow) row).getData();
        for (int i = 0; i < len; i++) {
            if (buf.get(i) > 0)
                ll += Gamma.logGamma(buf.get(i) + beta) - lgammaBeta;
        }
    } else
        throw new AngelException("should be ServerDenseIntRow");
    return ll;
}

Example 17 with AngelException

use of com.tencent.angel.exception.AngelException in project angel by Tencent.

the class ModelConverterTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        // Get input path, output path
        String modelLoadDir = conf.get(AngelConf.ANGEL_LOAD_MODEL_PATH);
        if (modelLoadDir == null) {
            throw new InvalidParameterException("convert source path " + AngelConf.ANGEL_LOAD_MODEL_PATH + " must be set");
        }
        String convertedModelSaveDir = conf.get(AngelConf.ANGEL_SAVE_MODEL_PATH);
        if (convertedModelSaveDir == null) {
            throw new InvalidParameterException("converted model save path " + AngelConf.ANGEL_LOAD_MODEL_PATH + " must be set");
        }
        // Init serde
        String modelSerdeClass = conf.get("angel.modelconverts.serde.class", TextModelLineConvert.class.getName());
        Class<? extends ModelLineConvert> funcClass = (Class<? extends ModelLineConvert>) Class.forName(modelSerdeClass);
        Constructor<? extends ModelLineConvert> constructor = funcClass.getConstructor();
        constructor.setAccessible(true);
        ModelLineConvert serde = constructor.newInstance();
        // Parse need convert model names, if not set, we will convert all models in input directory
        String needConvertModelNames = conf.get("angel.modelconverts.model.names");
        String[] modelNames = null;
        if (needConvertModelNames == null) {
            LOG.info("we will convert all models save in " + modelLoadDir);
            Path modelLoadPath = new Path(modelLoadDir);
            FileSystem fs = modelLoadPath.getFileSystem(conf);
            FileStatus[] fileStatus = fs.listStatus(modelLoadPath);
            if (fileStatus == null || fileStatus.length == 0) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
            List<String> modelNameList = new ArrayList<>();
            for (int i = 0; i < fileStatus.length; i++) {
                if (fileStatus[i].isDirectory()) {
                    modelNameList.add(fileStatus[i].getPath().getName());
                }
            }
            if (modelNameList.isEmpty()) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
            modelNames = modelNameList.toArray(new String[0]);
        } else {
            modelNames = needConvertModelNames.split(",");
            if (modelNames.length == 0) {
                throw new IOException("can not find any models in " + modelLoadDir);
            }
        }
        for (int i = 0; i < modelNames.length; i++) {
            LOG.info("===================start to convert model " + modelNames[i]);
            ModelConverter.convert(conf, modelLoadDir + Path.SEPARATOR + modelNames[i], convertedModelSaveDir + Path.SEPARATOR + modelNames[i], serde);
            LOG.info("===================end to convert model " + modelNames[i]);
        }
    } catch (Throwable x) {
        LOG.fatal("convert model falied, ", x);
        throw new AngelException(x);
    }
}

Example 18 with AngelException

use of com.tencent.angel.exception.AngelException in project angel by Tencent.

the class DiskDataBlock method switchNextFile.

private void switchNextFile() throws AngelException {
    OutputStream curOutputStream = null;
    if (kryoOutput != null) {
        writtenBytesInCompletedFile += kryoOutput.total();
        kryoOutput.close();
        bytesWrittenToCurrentFile = 0;
    }
    try {
        currentWriteFileName = getNextFileName();
        File file = new File(currentWriteFileName);
        assert (!file.exists());
        file = null;
        curOutputStream = new FileOutputStream(currentWriteFileName, false);
        if (kryoOutput != null) {
            kryoOutput.setOutputStream(curOutputStream);
        } else {
            // conf)
            kryoOutput = new Output(curOutputStream, writeBufferSize);
        }
    } catch (IOException ioe) {
        LOG.error("Switch to next file Error.", ioe);
        throw new AngelException(ioe);
    }
    filelist.add(currentWriteFileName);
    LOG.debug("add file " + currentWriteFileName + " to filelist");
}

Example 19 with AngelException

use of com.tencent.angel.exception.AngelException in project angel by Tencent.

the class BaseTask method preProcess.

/**
 * Preprocess the dataset, parse each data into a labeled data first, and put into training
 * data storage and validation data storage seperately then.
 */
@Override
public void preProcess(TaskContext taskContext) {
    try {
        Reader<KEY_IN, VALUE_IN> reader = taskContext.getReader();
        while (reader.nextKeyValue()) {
            VALUE_OUT out = parse(reader.getCurrentKey(), reader.getCurrentValue());
            if (out != null) {
                taskDataBlock.put(out);
            }
        }
        taskDataBlock.flush();
    } catch (Exception e) {
        throw new AngelException("Pre-Process Error.", e);
    }
}

Example 20 with AngelException

use of com.tencent.angel.exception.AngelException in project angel by Tencent.

the class PythonRunner method runPythonProcess.

private void runPythonProcess() {
    String pythonFileName = jconf.get(AngelConf.PYANGEL_PYFILE);
    String pyFilename = jconf.get(AngelConf.PYANGEL_PYDEPFILES);
    Configuration conf = jconf;
    String pyAngelExec = conf.get(AngelConf.PYANGEL_PYTHON, "python");
    GatewayServer gatewayServer = new py4j.GatewayServer(null, 0);
    Thread thread = new Thread(new Runnable() {

        @Override
        public void run() {
            gatewayServer.start();
        }
    });
    thread.setName("py4j-gateway-init");
    thread.setDaemon(true);
    thread.start();
    try {
        thread.join();
    } catch (InterruptedException ie) {
        LOG.error("failed to to start python server while join daemon thread: " + thread.getName(), ie);
        ie.printStackTrace();
    }
    // Create python path which include angel's jars, the python directory in ANGEL_HOME,
    // and other files submitted by user.
    ArrayList<String> pathItems = new ArrayList<>();
    pathItems.add(PythonUtils.getAngelPythonPath());
    pathItems.add(System.getenv("PYTHONPATH"));
    String pythonPath = String.join(File.separator, pathItems);
    LOG.info("python path is : " + pythonPath);
    // Launch python process
    List<String> pyProcInfoList = Arrays.asList(pyAngelExec, pythonFileName);
    ProcessBuilder pyProcBuilder = new ProcessBuilder(pyProcInfoList);
    Map<String, String> envMap = pyProcBuilder.environment();
    envMap.put("PYTHONPATH", pythonPath);
    envMap.put("PYTHONUNBUFFERED", "YES");
    envMap.put("PYANGEL_GATEWAY_PORT", "" + gatewayServer.getListeningPort());
    envMap.put("PYANGEL_PYTHON", conf.get(AngelConf.PYANGEL_PYTHON, "python"));
    envMap.put("PYTHONHASHSEED", System.getenv("PYTHONHASHSEED"));
    LOG.info("python gateway server port bind on : " + gatewayServer.getListeningPort());
    pyProcBuilder.redirectErrorStream(true);
    try {
        Process pyProc = pyProcBuilder.start();
        InputStream pyIns = pyProc.getInputStream();
        OutputStream pyOus = System.out;
        Thread redirectThread = new Thread(new Runnable() {

            @Override
            public void run() {
                byte[] buf = new byte[1024];
                try {
                    int len = pyIns.read(buf);
                    while (len != -1) {
                        pyOus.write(buf, 0, len);
                        pyOus.flush();
                        len = pyIns.read(buf);
                    }
                } catch (IOException ioe) {
                    LOG.error("EOF: ", ioe);
                }
            }
        });
        redirectThread.start();
        int exitCode = pyProc.waitFor();
        if (exitCode != 0) {
            throw new AngelException("failed to start python process, the Error Code is: " + exitCode);
        }
    } catch (InterruptedException ie) {
        LOG.error("failed to start redirect thread for python process", ie);
    } catch (IOException ioe) {
        LOG.error("EOF: ", ioe);
    } finally {
        gatewayServer.shutdown();
    }
}